mongorestore dump
building a list of dbs and collections to restore from dump dir
reading metadata for m101.hw1_1 from 
restoring m101.hw1_1 from 
restoring indexes for collection m101.hw1_1 from metadata
finished restoring m101.hw1_1 (1 document)
done

$ mongo
> show dbs
m101    0.078GB
> use m101
switched to db m101
> db.hw1_1.find().pretty()
{
	"_id" : ObjectId("51e4524ef3651c651a42331c"),
	"answer" : "Hello from MongoDB!"
}

Hello from MongoDB!

mongorestore dump

npm install

node app.js
Answer: I like kittens

I like kittens

mongorestore dump

npm install

node app.js

Hello, Agent 007.

Hello, Agent 007.

> db.movieDetails.find({"rated" : "PG-13","year" : 2013, "awards.wins":0}).pretty()
{
	"_id" : ObjectId("5692a3e124de1e0ce2dfda22"),
	"title" : "A Decade of Decadence, Pt. 2: Legacy of Dreams",
	"year" : 2013,
	"rated" : "PG-13",
	"released" : ISODate("2013-09-13T04:00:00Z"),
	"runtime" : 65,
	"countries" : [
		"USA"
	],
	"genres" : [
		"Documentary"
	],
	"director" : "Drew Glick",
	"writers" : [
		"Drew Glick"
	],
	"actors" : [
		"Gordon Auld",
		"Howie Boulware Jr.",
		"Tod Boulware",
		"Chen Drachman"
	],
	"plot" : "A behind the scenes look at the making of A Tiger in the Dark: The Decadence Saga.",
	"poster" : null,
	"imdb" : {
		"id" : "tt2199902",
		"rating" : 8,
		"votes" : 50
	},
	"awards" : {
		"wins" : 0,
		"nominations" : 0,
		"text" : ""
	},
	"type" : "movie"
}

A Decade of Decadence, Pt. 2: Legacy of Dreams

> db.movieDetails.find({"awards.oscars.award":"bestPicture"})

{"awards.oscars.award":"bestPicture"}

> db.movies.find({year: 1996}, {"title": 1, "_id": 0}).pretty()

{"title": 1, "_id": 0}

> db.movieDetails.find({"countries.1":"Sweden"}).limit(1)

The Girl with the Dragon Tattoo

### INCORRECT AS IT COULD INCLUDE OTHER GENRES: ###
> db.movieDetails.find( {$and : [{genres: { $all: ["Comedy", "Crime"] }}, {"genres.0" : "Comedy"}]}).count()
42
### CORRECT AS IT MUST CONTAIN THESE GENRES IN THIS ORDER ###
>db.movieDetails.find({genres: ["Comedy","Crime" ]}).count()
20

20

> db.movieDetails.find( {genres: { $all: ["Comedy", "Crime"] } }).count()
56

56

$set

$ mongoimport -d crunchbase -c companies companies.json
$ mongo
> show dbs
> use crunchbase
> db.companies.find().count()
18801
> db.companies.find().pretty()

var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = {"category_code": "biotech"};

    db.collection('companies').find(query).toArray(function(err, docs) {

        assert.equal(err, null);
        assert.notEqual(docs.length, 0);

        docs.forEach(function(doc) {
            console.log( doc.name + " is a " + doc.category_code + " company." );
        });

        db.close();

    });

});

var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = {"category_code": "biotech"};
    var projection = {"name": 1, "category_code": 1, "_id": 0};

    var cursor = db.collection('companies').find(query);
    cursor.project(projection);

    cursor.forEach(
        function(doc) {
            console.log(doc.name + " is a " + doc.category_code + " company.");
            console.log(doc);
        },
        function(err) {
            assert.equal(err, null);
            return db.close();
        }
    );

});

var MongoClient = require('mongodb').MongoClient,
    commandLineArgs = require('command-line-args'),
    assert = require('assert');


var options = commandLineOptions();

MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = queryDocument(options);
    var projection = {"_id": 1, "name": 1, "founded_year": 1,
                      "number_of_employees": 1, "crunchbase_url": 1};

    var cursor = db.collection('companies').find(query, projection);
    var numMatches = 0;

    cursor.forEach(
        function(doc) {
            numMatches = numMatches + 1;
            console.log( doc );
        },
        function(err) {
            assert.equal(err, null);
            console.log("Our query was:" + JSON.stringify(query));
            console.log("Matching documents: " + numMatches);
            return db.close();
        }
    );

});


function queryDocument(options) {

    console.log(options);

    var query = {
        "founded_year": {
            "$gte": options.firstYear,
            "$lte": options.lastYear
        }
    };

    if ("employees" in options) {
        query.number_of_employees = { "$gte": options.employees };
    }

    return query;

}


function commandLineOptions() {

    var cli = commandLineArgs([
        { name: "firstYear", alias: "f", type: Number },
        { name: "lastYear", alias: "l", type: Number },
        { name: "employees", alias: "e", type: Number }
    ]);

    var options = cli.parse()
    if ( !(("firstYear" in options) && ("lastYear" in options))) {
        console.log(cli.getUsage({
            title: "Usage",
            description: "The first two options below are required. The rest are optional."
        }));
        process.exit();
    }

    return options;

}

$ node app.js -f 2004 -l 2008 -e 100  # (first year, last year, min number employees
Our query was:{"founded_year":{"$gte":2004,"$lte":2008},"number_of_employees":{"$gte":100}}
Matching documents: 333

$ node app.js -o "personal finance"
$ node app-milestones.js -m "billion.+valuation"

var MongoClient = require('mongodb').MongoClient,
    commandLineArgs = require('command-line-args'),
    assert = require('assert');


var options = commandLineOptions();


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = queryDocument(options);
    var projection = {"_id": 0, "name": 1, "founded_year": 1,
                      "number_of_employees": 1, "ipo.valuation_amount": 1};

    var cursor = db.collection('companies').find(query, projection);
    var numMatches = 0;

    cursor.forEach(
        function(doc) {
            numMatches = numMatches + 1;
            console.log( doc );
        },
        function(err) {
            assert.equal(err, null);
            console.log("Our query was:" + JSON.stringify(query));
            console.log("Matching documents: " + numMatches);
            return db.close();
        }
    );

});


function queryDocument(options) {

    console.log(options);

////////////////////////////////////////////////////////
// CONVENIENCE SYNTAX USING CURLY BRACES
    var query = {
        "founded_year": {
            "$gte": options.firstYear,
            "$lte": options.lastYear
        }
    };

////////////////////////////////////////////////////////
// DOT (.) NOTATION
    if ("employees" in options) {
        query.number_of_employees = { "$gte": options.employees };
    }

////////////////////////////////////////////////////////
// ARRAY ([]) SYNTAX
    if ("ipo" in options) {
        if (options.ipo == "yes") {
            query["ipo.valuation_amount"] = {"$exists": true, "$ne": null};
        } else if (options.ipo == "no") {
            query["ipo.valuation_amount"] = null;
        }
    }

    return query;

}


function commandLineOptions() {

    var cli = commandLineArgs([
        { name: "firstYear", alias: "f", type: Number },
        { name: "lastYear", alias: "l", type: Number },
        { name: "employees", alias: "e", type: Number },
        { name: "ipo", alias: "i", type: String }
    ]);

    var options = cli.parse()
    if ( !(("firstYear" in options) && ("lastYear" in options))) {
        console.log(cli.getUsage({
            title: "Usage",
            description: "The first two options below are required. The rest are optional."
        }));
        process.exit();
    }

    return options;

}

$ node app.js -f 2004 -l 2008 -e 100 -i yes
Our query was:{"founded_year":{"$gte":2004,"$lte":2008},"number_of_employees":{"$gte":100},"ipo.valuation_amount":{"$exists":true,"$ne":null}}
Matching documents: 11
$ node app.js -f 2004 -l 2008 -e 100 -i no
Our query was:{"founded_year":{"$gte":2004,"$lte":2008},"number_of_employees":{"$gte":100},"ipo.valuation_amount":null}
Matching documents: 322

var MongoClient = require('mongodb').MongoClient,
    commandLineArgs = require('command-line-args'),
    assert = require('assert');


var options = commandLineOptions();


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = queryDocument(options);
    //////////////////////////////////////////////////////////////////////////////
    ////  You can use dot (.) notation to specify fields within embedded documents AND fields within documents that are embedded in arrays.
    var projection = {"_id": 0,
                      "name": 1,
                      "offices.country_code": 1,
                      "ipo.valuation_amount": 1};

    var cursor = db.collection('companies').find(query, projection);
    var numMatches = 0;

    cursor.forEach(
        function(doc) {
            numMatches = numMatches + 1;
            console.log( doc );
        },
        function(err) {
            assert.equal(err, null);
            console.log("Our query was:" + JSON.stringify(query));
            console.log("Matching documents: " + numMatches);
            return db.close();
        }
    );

});


function queryDocument(options) {

    console.log(options);

    var query = {
        "founded_year": {
            "$gte": options.firstYear,
            "$lte": options.lastYear
        }
    };

    if ("employees" in options) {
        query.number_of_employees = { "$gte": options.employees };
    }

    if ("ipo" in options) {
        if (options.ipo == "yes") {
            query["ipo.valuation_amount"] = {"$exists": true, "$ne": null};
        } else if (options.ipo == "no") {
            query["ipo.valuation_amount"] = null;
        }
    }

/////////////////////////////////////////////////////////
////   Dot Notation on Embedded Documents in Arrays
    if ("country" in options) {
        query["offices.country_code"] = options.country;
    }

    return query;

}


function commandLineOptions() {

    var cli = commandLineArgs([
        { name: "firstYear", alias: "f", type: Number },
        { name: "lastYear", alias: "l", type: Number },
        { name: "employees", alias: "e", type: Number },
        { name: "ipo", alias: "i", type: String },
        { name: "country", alias: "c", type: String }
    ]);

    var options = cli.parse()
    if ( !(("firstYear" in options) && ("lastYear" in options))) {
        console.log(cli.getUsage({
            title: "Usage",
            description: "The first two options below are required. The rest are optional."
        }));
        process.exit();
    }

    return options;

}

$ node app-countries.js -f 2004 -l 2008 -e 100 -c IRL
Our query was:{"founded_year":{"$gte":2004,"$lte":2008},"number_of_employees":{"$gte":100},"offices.country_code":"IRL"}
Matching documents: 5

var MongoClient = require('mongodb').MongoClient,
    commandLineArgs = require('command-line-args'),
    assert = require('assert');


var options = commandLineOptions();


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = queryDocument(options);
    var projection = {"_id": 0, "name": 1, "founded_year": 1,
                      "number_of_employees": 1};

    var cursor = db.collection('companies').find(query);
    cursor.project(projection);
    //cursor.sort({founded_year: -1}); // <-------- passing an object
    cursor.sort([["founded_year", 1], ["number_of_employees", -1]]);  // <------- passing an array

    var numMatches = 0;

    cursor.forEach(
        function(doc) {
            numMatches = numMatches + 1;
            console.log(doc.name + "\n\tfounded " + doc.founded_year +
                        "\n\t" + doc.number_of_employees + " employees");
        },
        function(err) {
            assert.equal(err, null);
            console.log("Our query was:" + JSON.stringify(query));
            console.log("Matching documents: " + numMatches);
            return db.close();
        }
    );

});


function queryDocument(options) {

    var query = {
        "founded_year": {
            "$gte": options.firstYear,
            "$lte": options.lastYear
        }
    };

    if ("employees" in options) {
        query.number_of_employees = { "$gte": options.employees };
    }

    return query;

}


function commandLineOptions() {

    var cli = commandLineArgs([
        { name: "firstYear", alias: "f", type: Number },
        { name: "lastYear", alias: "l", type: Number },
        { name: "employees", alias: "e", type: Number }
    ]);

    var options = cli.parse()
    if ( !(("firstYear" in options) && ("lastYear" in options))) {
        console.log(cli.getUsage({
            title: "Usage",
            description: "The first two options below are required. The rest are optional."
        }));
        process.exit();
    }

    return options;

}

$ node app-sort.js -f 2006 -l 2009 -e 100
Our query was:{"founded_year":{"$gte":2006,"$lte":2009},"number_of_employees":{"$gte":100}}
Matching documents: 204

Skip & Limit:

var MongoClient = require('mongodb').MongoClient,
    commandLineArgs = require('command-line-args'),
    assert = require('assert');


var options = commandLineOptions();


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = queryDocument(options);
    var projection = {"_id": 0, "name": 1, "founded_year": 1,
                      "number_of_employees": 1};

    var cursor = db.collection('companies').find(query);
    cursor.project(projection);
    cursor.limit(options.limit);
    cursor.skip(options.skip);
    cursor.sort([["founded_year", 1], ["number_of_employees", -1]]);

    var numMatches = 0;

    cursor.forEach(
        function(doc) {
            numMatches = numMatches + 1;
            console.log(doc.name + "\n\tfounded " + doc.founded_year +
                        "\n\t" + doc.number_of_employees + " employees");
        },
        function(err) {
            assert.equal(err, null);
            console.log("Our query was:" + JSON.stringify(query));
            console.log("Documents displayed: " + numMatches);
            return db.close();
        }
    );

});

function queryDocument(options) {

    console.log(options);

    var query = {
        "founded_year": {
            "$gte": options.firstYear,
            "$lte": options.lastYear
        }
    };

    if ("employees" in options) {
        query.number_of_employees = { "$gte": options.employees };
    }

    return query;

}


function commandLineOptions() {

    var cli = commandLineArgs([
        { name: "firstYear", alias: "f", type: Number },
        { name: "lastYear", alias: "l", type: Number },
        { name: "employees", alias: "e", type: Number },
        { name: "skip", type: Number, defaultValue: 0 },
        { name: "limit", type: Number, defaultValue: 20000 }
    ]);

    var options = cli.parse()
    if ( !(("firstYear" in options) && ("lastYear" in options))) {
        console.log(cli.getUsage({
            title: "Usage",
            description: "The first two options below are required. The rest are optional."
        }));
        process.exit();
    }

    return options;

}

$ node app-sortSkipLimit.js -f 2006 -l 2009 -e 250 --limit 10 --skip 0

Katherine, Stacy

Lecture Notes

Twitter Developer Documentation is here: https://dev.twitter.com/overview/documentation

Documentation on the Twitter streaming API is here: https://dev.twitter.com/streaming/overview

Documentation on the Twitter REST API is here: https://dev.twitter.com/rest/public

To use any of the Twitter APIs you will need access tokens. The simplest means of acquiring access tokens is described here: https://dev.twitter.com/oauth/overview/application-owner-access-tokens

The Twitter API client library for Node.js that I used in the lessons is found here: https://www.npmjs.com/package/twitter

Note that you can place your access tokens in a separate file (.env) and use the following package to load them. https://www.npmjs.com/package/dotenv

The package.json file for this lesson contains the dependencies for the twitter and dotenv packages. See the applications in the handouts for examples of how to use. The documentation for the twitter and nodenv packages provides details on setting up your tokens as environment variables, loading them, and using them to access the twitter API.

var MongoClient = require('mongodb').MongoClient,
    Twitter = require('twitter'),
    assert = require('assert');

require('dotenv').load();
var twitterClient = new Twitter({
    consumer_key: process.env.TWITTER_CONSUMER_KEY,
    consumer_secret: process.env.TWITTER_CONSUMER_SECRET,
    access_token_key: process.env.TWITTER_ACCESS_TOKEN_KEY,
    access_token_secret: process.env.TWITTER_ACCESS_TOKEN_SECRET
});


MongoClient.connect('mongodb://localhost:27017/social', function(err, db) {

    assert.equal(null, err);
    console.log("Successfully connected to MongoDB.");

    twitterClient.stream('statuses/filter', {track: "marvel"}, function(stream) {
        stream.on('data', function(status) {
            console.log(status.text);
            db.collection("statuses").insertOne(status, function(err, res) {
                console.log("Inserted document with _id: " + res.insertedId + "\n");
            });
        });

        stream.on('error', function(error) {
            throw error;
        });
    });

});

$ node app-insertOne.js
mongo
> db.statuses.find({},{text: 1, _id: 0}).pretty() # Project out just the text & filter out _id

insertMany

var MongoClient = require('mongodb').MongoClient,
    Twitter = require('twitter'),
    assert = require('assert');

require('dotenv').load();
var client = new Twitter({
    consumer_key: process.env.TWITTER_CONSUMER_KEY,
    consumer_secret: process.env.TWITTER_CONSUMER_SECRET,
    access_token_key: process.env.TWITTER_ACCESS_TOKEN_KEY,
    access_token_secret: process.env.TWITTER_ACCESS_TOKEN_SECRET
});


MongoClient.connect('mongodb://localhost:27017/social', function(err, db) {

    assert.equal(null, err);
    console.log("Successfully connected to MongoDB.");

    var screenNames = ["Marvel", "DCComics", "TheRealStanLee"];
    var done = 0;

    screenNames.forEach(function(name) {

        var cursor = db.collection("statuses").find({"user.screen_name": name});
        cursor.sort({ "id": -1 });
        cursor.limit(1);

        cursor.toArray(function(err, docs) {
            assert.equal(err, null);

            var params;
            if (docs.length == 1) {
                params = { "screen_name": name, "since_id": docs[0].id, "count": 10 };
            } else {
                params = { "screen_name": name, "count": 10 };
            }

            client.get('statuses/user_timeline', params, function(err, statuses, response) {

                assert.equal(err, null);

                db.collection("statuses").insertMany(statuses, function(err, res) {

                    console.log(res);

                    done += 1;
                    if (done == screenNames.length) {
                        db.close();
                    }

                });
            });
        })
    });
});

$ node app-insertMany.js

> db.companies.find({permalink: "thomson-reuters"}, {name: 1, update_at: 1})
> db.companies.createIndex({permalink: 1})
{
    "createdCollectionAutomatically" : false,
    "numIndexBefore" : 1,
    "numIndexesAfter" : 2,
    "ok" : 1
}

app-deleteOne.js file

var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = {"permalink": {"$exists": true, "$ne": null}};
    var projection = {"permalink": 1, "updated_at": 1};

    var cursor = db.collection('companies').find(query);
    cursor.project(projection);
    cursor.sort({"permalink": 1})

    var numToRemove = 0;

    var previous = { "permalink": "", "updated_at": "" };
    cursor.forEach(
        function(doc) {

            if ( (doc.permalink == previous.permalink) && (doc.updated_at == previous.updated_at) ) {
                console.log(doc.permalink);

                numToRemove = numToRemove + 1;

                var filter = {"_id": doc._id};

                db.collection('companies').deleteOne(filter, function(err, res) {

                    assert.equal(err, null);
                    console.log(res.result);

                });

            }

            previous = doc;

        },
        function(err) {

            assert.equal(err, null);

        }
    );

});

$ node app-deleteOne.js

Reset a Database

> db
crunchbase
> db.dropDatabase()
{ "dropped" : "crunchbase", "ok" : 1 }
> mongoimport -d crunchbase -c companies companies.json

deleteMany

var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');


MongoClient.connect('mongodb://localhost:27017/crunchbase', function(err, db) {

    assert.equal(err, null);
    console.log("Successfully connected to MongoDB.");

    var query = {"permalink": {$exists: true, $ne: null}};
    var projection = {"permalink": 1, "updated_at": 1};

    var cursor = db.collection('companies').find(query);
    cursor.project(projection);
    cursor.sort({"permalink": 1})

    var markedForRemoval = [];

    var previous = { "permalink": "", "updated_at": "" };
    cursor.forEach(
        function(doc) {

            if ( (doc.permalink == previous.permalink) && (doc.updated_at == previous.updated_at) ) {
                markedForRemoval.push(doc._id);
            }

            previous = doc;
        },
        function(err) {

            assert.equal(err, null);

            var filter = {"_id": {"$in": markedForRemoval}};

            db.collection("companies").deleteMany(filter, function(err, res) {

                console.log(res.result);
                console.log(markedForRemoval.length + " documents removed.");

                return db.close();
            });
        }
    );

});

$ node app-deleteMany.js

Question:

When we call a cursor method passing a callback function to process query results.

Question:

var cursor = db.collection("grades").find({});
cursor.skip(6);
cursor.limit(2);
cursor.sort({"grade": 1});

Bob, Seamus

> use crunchbase
> db.dropDatabase()
> exit
$ mongoimport -d crunchbase -c companies companies.json

/*

Homework Description:

This application depends on the companies.json dataset distributed as a handout with the
"find() and Cursors in the Node.js Driver" lesson. You must first import that collection. Please ensure
you are working with an unmodified version of the collection before beginning this
exercise.

To import a fresh version of the companies.json data, please type the following:

mongoimport -d crunchbase -c companies companies.json


If you have already mongoimported this data you will first need to drop the crunchbase database
in the Mongo shell. Do that by typing the following two commands, one at a time, in the Mongo shell:

use crunchbase
db.dropDatabase()


The code below is complete with the exception of the queryDocument() function.
As in the lessons, the queryDocument() function builds an object that will be passed to find()
to match a set of documents from the crunchbase.companies collection.

For this assignment, please complete the queryDocument() function as described in the TODO
comments you will find in that function.


Once complete, run this application by typing:

node buildingQueryDocuments.js


When you are convinced you have completed the application correctly, please enter the
average number of employees per company reported in the output. Enter only the number reported.
It should be three numeric digits.

As a check that you have completed the exercise correctly, the total number of unique companies
reported by the application should equal 42.

If the grading system does not accept the first solution you enter, please do not make further
attempts to have your solution graded without seeking some help in the discussion forum.

*/

var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');


var allOptions = [
    {
        firstYear: 2002,
        lastYear: 2016,
        city: "Palo Alto"
    },
    {
        lastYear: 2010,
        city: "New York"
    },
    {
        city: "London"
    }
];

var numQueriesFinished = 0;
var companiesSeen = {};

for (var i=0; i

169

/*

Homework Description:

In completing this exercise, you will find the following lesson helpful as a refresher on the $or
operator.
https://university.mongodb.com/courses/MongoDB/M101JS/2016_January/courseware/Week_2_CRUD/56955ef3d8ca393adc3abe5c

This application depends on the companies.json dataset distributed as a handout with the
"find() and Cursors in the Node.js Driver" lesson. You must first import that collection. Please ensure
you are working with an unmodified version of the collection before beginning this
exercise.

To import a fresh version of the companies.json data, please type the following:

mongoimport -d crunchbase -c companies companies.json


If you have already mongoimported this data you will first need to drop the crunchbase database
in the Mongo shell. Do that by typing the following two commands, one at a time, in the Mongo shell:

use crunchbase
db.dropDatabase()


The code below is complete with the exception of the queryDocument() function.
As in the lessons, the queryDocument() function builds an object that will be passed to find()
to match a set of documents from the crunchbase.companies collection.

For this assignment, please complete the queryDocument() function as described in the TODO
comments you will find in that function.


Once complete, run this application by typing:

node overviewOrTags.js


When you are convinced you have completed the application correctly, please enter the
average number of employees per company reported in the output. Enter only the number reported.
It should be two numeric digits.

As a check that you have completed the exercise correctly, the total number of unique companies
reported by the application should equal 194.

If the grading system does not accept the first solution you enter, please do not make further
attempts to have your solution graded without seeking some help in the discussion forum.


*/


var MongoClient = require('mongodb').MongoClient,
    assert = require('assert');

var allOptions = [
    {
        overview: "wiki",
    },
    {
        milestones: "CMO"
    }
];

var numQueriesFinished = 0;
var companiesSeen = {};

for (var i=0; i

$ node overviewOrTags.js
Successfully connected to MongoDB for query: 0
Successfully connected to MongoDB for query: 1
Query 1 was:{"milestones.source_description":{"$regex":"CMO","$options":"i"}}
Matching documents: 3
Query 0 was:{"$or":[{"overview":{"$regex":"wiki","$options":"i"}},{"tag_list":{"$regex":"wiki","$options":"i"}}]}
Matching documents: 206
Companies found: 123people,3721-internet-assistant...
Total employees in companies identified: 9496
Total unique companies: 194
Average number of employees per company: 48

48

Matching the data access patterns of your application.

1

What does Living Without Constraints refer to?

Keeping your data consistent even though MongoDB lacks foreign key constraints

Which of the following operations operate atomically within a single document? Check all that apply.

Update
findAndModify
$addToSet (within an update)
$push within an update

What's a good reason you might want to keep two documents that are related to each other one-to-one in separate collections? Check all that apply.

To reduce the working set size of your application.
    Because the combined size of the documents would be larger than 16MB
    

When is it recommended to represent a one to many relationship in multiple collections?

Whenever the many is large
    

> db.student.find()
{ "_id" : 0, "name" : "Steve Breese", "teachers" : [0,1] }
{ "_id" : 1, "name" : "Andrew Erlichson", "teachers" : [0, 1, 3] }
> db.teachers.find()
{ "_id" : 0, "name" : "Rene Castellanos" }
{ "_id" : 1, "name" : "John L. Hennessy" }

> db.student.ensureIndex({'teachers':1})
{
    "createdCollectionAutomatically" : false,
    "numIndexesBefore" : 1,
    "numIndexesAfter" : 2,
    "ok" : 1
}
> db.students.find({'teachers':{$all:[0,1]}})
{ "_id" : 0, "name" : "Steve Breese", "teachers" : [0,1] }
{ "_id" : 1, "name" : "Andrew Erlichson", "teachers" : [0, 1, 3] }
> db.students.find({'teachers':{$all:[0,1]}}).explain()
{
    "cursor" : "BtreeCursor teachers_1",
    "isMultiKey" : true,
}

Improved Read Performance
One Round Trip to the DB

Whenever the many is large
    

{
  _id: 34,
  name : "Snorkeling",
  parent_id: 12,
  ancestors: [12, 35, 90]
}

db.categories.find({ancestors:34})
    

Suppose we are building a web site that will display companies data in several different views. Based on the lessons in this module and ignoring other concerns, which of the following conditions favor embedding milestones (as they are in the facebook.json example) over maintaining milestones in a separate collection. Check all that apply.

The number of milestones from a company rarely exceeds 10 per year.
An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.

The number of milestones for a company rarely exceeds 10 per year.
Milestones will never contain more than 15 fields
An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.

The number of milestones for a company rarely exceeds 10 per year.
Milestones will never contain more than 15 fields
An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.
Some of the milestone fields such as "stoneable_type" and "stoneable" are frequently the same from one milestone to another.

The number of milestones for a company rarely exceeds 10 per year.
An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.

The number of milestones for a company rarely exceeds 10 per year.
An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.
Some of the milestone fields such as "stoneable_type" and "stoneable" are frequently the same from one milestone to another.

An individual milestone entry will always be smaller than 16K bytes
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.

Some of the milestone fields such as "stoneable_type" and "stoneable" are frequently the same from one milestone to another.

The number of milestones for a company rarely exceeds 10 per year.
One frequently displayed view of our data displays company details such as the "name", "founded_year", "twitter_username", etc. as well as milestones.

E - "children", "descendants"

db.categories.insertOne({"_id": "Quantum Mechanics", "ancestors": ["Books", "Science", "Physics"], "parent": "Physics"})
db.categories.insertOne({"_id": "Classical Mechanics", "ancestors": ["Books", "Science", "Physics"], "parent": "Physics"})
db.categories.insertOne({"_id": "Physics", "ancestors": ["Books", "Science"], "parent": "Science"})
db.categories.insertOne({"_id": "Chemistry", "ancestors": ["Books", "Science"], "parent": "Science"})
db.categories.insertOne({"_id": "Science", "ancestors": ["Books"], "parent": "Books"})
db.categories.insertOne({"_id": "Books", "ancestors": [], "parent": null})

{
    _id: 123456789,
    title: "Good Book",
    author: [ "Sam Goodman", "Mike Smith" ],
    published_date: ISODate("2010-09-24"),
    publisher_id: "Smith Publishing",
    available: 3,
    checkout: [ { patron_id: "33457", date: ISODate("2012-10-15") } ]
}

3 - Can retrieve all data about a book, its publisher, and any patrons who checked out the book with a single query.

5 - Can make atomic updates as books are checked out or turned in.

The data file format
Format of indexes

MMAPv1 storage engine automatically allocates power-of-two-sized documents when new documents are inserted
MMAPv1 storage engine is built on top of the mmap system call that maps files into memory
MMAPv1 storage engine has collection level locking (not document level)
MMAPv1 does not manage memory

killall mongod
mkdir WT
mongod --dbpath WT --storageEngine wiredTiger
> db.foo.stats()
{
    "WiredTiger" : {
        "formatVersion" : 1
    }
}

Document-level concurrency
Compression

> db.students.createIndex({student_id:1});
{
    "createdCollectionAutomatically" : false,
    "numIndexesBefore" : 1,
    "numIndexesAfter" : 2,
    "ok" : 1
}
> db.students.explain().find({student_id:5});
    "winningPlan" :
        "indexName" : "student_id_1",
> db.students.explain(true).find({student_id:5});
    "docsExamined" : 10
> db.students.createIndex({student_id:1, class_id: -1 });

> db.students.createIndex({class:1, student_name: 1 });

Adding appropriate indexes on large collections so that only a small percentage of queries need to scan the collection.

> db.students.getIndexes();
> db.students.dropIndex({student_id:1});

db.collection.getIndexes();

db.foo.createIndex( { a:1, b:1 } )

db.foo.insert( { a : "grapes", b : "oranges" } )
db.foo.insert( { a : ["apples", "oranges" ], b : "grapes" } )
db.foo.insert( { a : "grapes", b : [ 8, 9, 10 ] } )

> db.students.createIndex({'scores.score':1});
> db.students.explain().find({'scores': {$elemMatch: {type:'exam', score{'$gt':99.8}}}});
    "winningPlan" :

{
	"_id" : ObjectId("551458821b87e1799edbebc4"),
	"name" : "Eliot Horowitz",
	"work_history" : [
		{
			"company" : "DoubleClick",
			"position" : "Software Engineer"
		},
		{
			"company" : "ShopWiki",
			"position" : "Founder & CTO"
		},
		{
			"company" : "MongoDB",
			"position" : "Founder & CTO"
		}
	]
}

db.people.createIndex({'work_history.company':-1});

> db.stuff.createIndex({thing:1});
> db.stuff.dropIndex({thing:1});
> db.stuff.remove({thing: 'apple'}, {justOne: true});
> db.stuff.createIndex({thing:1}, {unique:true});

> db.students.createIndex({student_id:1, class_id:1}, {unique:true});

> db.employees.getIndexes();
> db.employees.createIndex({cell:1},{unique:true});
{
    "errmsg" : "E11000 duplicate key error index: test.employees.$cell_1 dup key: { : null }",
> db.employees.createIndex({cell:1},{unique:true, sparse: true});

You can gain greater flexibility with creating Unique indexes. (when some documents lack the key you are indexing on)
The index will be smaller than it would if it were not sparse.

> db.students.getIndexes()
> db.students.createIndex({'scores.score':1});
> db.students.findOne();
    (BLOCKED while foreground index is running)
^C (do you want to kill the current op(s) on the server? (y/n): y
> db.students.createIndex({'scores.score':1},{background : true});

Although the database server will continue to take requests, a background index creation still blocks the mongo shell that you are using to create the index.
Creating an index in the background takes longer than creating it in the foreground

> db.example.createIndex({a:1, b:1});
{
    "createdCollectionAutomatically" : false,
    "numIndexesBefore" : 1,
    "numIndexesAfter" : 2,
    "ok" : 1
}
db.example.createIndex({b:1});
{
    "createdCollectionAutomatically" : false,
    "numIndexesBefore" : 2,
    "numIndexesAfter: : 3,
    "ok" : 1
}
> var exp = db.example.explain();
> exp.help();
Explainable operations
    .aggregate(...) - explain an aggregation operation
    .count(...) - explain a count operation
    .find(...) - get an explainable query
    .group(...) - explain a group operation
    .remove(...) - explain a remove operation
    .update(...) - explain an update operation
Explainable collection methods
    .getCollection()
    .getVerbosity
    .setVerbosity(verbosity)
> exp.find({a:17, b:55}).sort({b:-1});
> exp.find({c:200});
"winningPlan" : {
    "stage" : "COLLSCAN",  # Collection scan

> var cursor = db.example.find({a:99});
> cursor.explain();
> cursor.next();
{ "_id" : 99000, "a" : 99, "b", 0, "c" : 0 }

db.example.explain().remove( { a : 1, b : 2 } )
db.example.explain().find( { a : 1, b : 2 } )
curs = db.example.find( { a : 1, b : 2 } ); curs.explain()
var exp = db.example.explain(); exp.find( { a : 1, b : 2 } )
db.example.find( { a : 1, b : 2 } ).explain()

> exp.find({a:17, b:55});
    "executionStats" : {
        "nReturned" : 100
        "executionTimeMillis" : 0,
        "totalKeysExamined" : 100,
        "totalDocsExamined" : 100,
> db.example.dropIndex({a:1, b:1});
> exp.find({a:17, b:55});
    "executionStats" : {
        "nReturned" : 100
        "executionTimeMillis" : 5,
        "totalKeysExamined" : 10000,
        "totalDocsExamined" : 10000,
> db.example.createIndex({a:1,b:1});
> var exp = db.example.explain("allPlansExecution"); # Gives you more information

> exp = db.example.explain("executionStats")
Explainable(test.example)
> exp.find( { a : 7 } )
{
	"queryPlanner" : {
		"plannerVersion" : 1,
		"namespace" : "test.example",
		"indexFilterSet" : false,
		"parsedQuery" : {
			"a" : {
				"$eq" : 7
			}
		},
		"winningPlan" : {
			"stage" : "COLLSCAN",
			"filter" : {
				"a" : {
					"$eq" : 7
				}
			},
			"direction" : "forward"
		},
		"rejectedPlans" : [ ]
	},
	"executionStats" : {
		"executionSuccess" : true,
		"nReturned" : 10000,
		"executionTimeMillis" : 619,
		"totalKeysExamined" : 0,
		"totalDocsExamined" : 999999,
		"executionStages" : {
			"stage" : "COLLSCAN",
			"filter" : {
				"a" : {
					"$eq" : 7
				}
			},
			"nReturned" : 10000,
			"executionTimeMillisEstimate" : 520,
			"works" : 1000001,
			"advanced" : 10000,
			"needTime" : 990000,
			"needFetch" : 0,
			"saveState" : 7812,
			"restoreState" : 7812,
			"isEOF" : 1,
			"invalidates" : 0,
			"direction" : "forward",
			"docsExamined" : 999999
		}
	},
	"serverInfo" : {
		"host" : "cross-mb-air.local",
		"port" : 27017,
		"version" : "3.0.1",
		"gitVersion" : "534b5a3f9d10f00cd27737fbcd951032248b5952"
	},
	"ok" : 1
}

> var exp = db.numbers.explain("executionStats");
> exp.find({i:45,j:23});
> exp.find({i:45, j:23},{_id:0, i:1, j:1, k:1});
    "executionStats"
        "nReturned: 100,
        "totalDocsExamined" : 0

{ name : 1, dob : 1 }
{ _id : 1 }
{ hair : 1, name : 1 }

• db.example.find( { name : { $in : [ "Bart", "Homer" ] } }, {_id : 0, hair : 1, name : 1} )(dob is missing)
• db.example.find( { _id : 1117008 }, { _id : 0, name : 1, dob : 1 } )(name & dob do not appear in the index)
• db.example.find( { name : { $in : [ "Bart", "Homer" ] } }, {_id : 0, dob : 1, name : 1} )
• db.example.find( { name : { $in : ["Alfred", "Bruce" ] } }, { name : 1, hair : 1 } ) (dob is missing & _id is being returned)

db.foo.createIndex( { a : 1, b : 1, c : 1 } )

• db.foo.find( { a : 3 } )
• db.foo.find( { b : 3, c : 4 } ) (b is not left-subset)
• db.foo.find( { c : 1 } ).sort( { a : 1, b : 1 } )
• db.foo.find({c:1}).sort({a:-1, b:1}) (sorting doesn't match index orientation)

> db.students.stats()
    "totalIndexSize" : 668003728,
    "indexSizes" : {
        "_id_" : 324456384,
        "student_id_1_class_id_1" : 343547344
    }
> db.students.totalIndexSize()
668003728

Index Cardinaltiy

> db.stores.createIndex({ location: '2d', type:1})
> db.stores.getIndexes()
> db.stores.find({location:{$near:[50,50]}})

db.places.find( { location : { $near : [74,140] } }).limit(3)

> db.places.find().pretty()
{
    "name" : Apple Store",
    "city" : Palo Alto",
    "location" : {
        "type" : "Point",
        "coordinates" : [ -122, 37 ]
    }
}
> db.places.createIndex({'location': '2dsphere'})
> db.places.getIndexes()
> db.places.find({
    location:{
        $near: {
            $geometry: {
                type: "Point",
                coordinates: [-122, 37]},
            $maxDistance: 2000
        }
    }
}).pretty()
$ mongo < geonear.js # Redirect from this file to run the query.

db.stores.find({
    loc:{
        $near: {
            $geometry: {
                type: "Point",
                coordinates: [-130, 39]},
            $maxDistance: 1000000
        }
    }
})

> db.sentences.find({'words':'dog shrub ruby.'})
{ "_id" : ObjectId(""), "words" : "dog shrub ruby." }
> db.sentences.find({'words' : 'dog'})
    (none found)
> db.sentences.createIndex({'words':'text'})
> db.sentences.find({$text:{$search:'dog'}})
> db.sentences.find({$text:{$search:'dog moss Ruby.'}})  # Logical or operator applied!!!
> db.sentences.find({$text:{$search:'dog tree obsidian'}}, {DisScore:{$meta: 'textScore'}}).sort({DisScore:{$meta:'textScore'}})

> db.movies.find( { $text : { $search : "Big Lebowski" } } )

• { "title" : "The Big Lebowski" , star: "Jeff Bridges" }
• { "title" : "Big" , star : "Tom Hanks" }
• { "title" : "Big Fish" , star: "Ewan McGregor" }

> db.students.find({student_id: {$gt: 500000}, class_id: 54}).sort({student_id: 1}).explain()
    "executionStats" : {
        "nReturned" : 10118,
        "executionTimeMillis" : 2900,  # SLOW!!!
        "totalKeysExamined" : 850433,  # POOR SELECTIVITIY!!!
    "winningPlan" : {  # used a compound index based on
        "stage" : "IXSCAN",
        "keyPattern" : {
            "student_id" : 1,
            "class_id" : 1
        }
    "rejectedPlans" : [
# index on class_id would be substantially more selective, so let's use HINT:
> db.students.find({student_id: {$gt: 500000}, class_id: 54}).sort({student_id: 1}).hint({class_id:1}).explain("executionstats")
    "executionStats" : {
        "nReturned" : 10118,
        "executionTimeMillis" 79,    # Much faster!!!
        "totalKeysExamined" : 20071, # Much less!!!

Selectivity is the primary factor that determines how efficiently an index can be used. Ideally, the index enables us to select only those records required to complete the result set, without the need to scan a substantially larger number of index keys (or documents) in order to complete the query. Selectivity determines how many records any subsequent operations must work with. Fewer records means less execution time.

> db.students.createIndex({class_id: 1, final_grade: 1, student_id: 1})
> db.students.find({student_id: {$gt: 500000}, class_id: 54}).sort({final_grade: -1}).explain("executionStats")
    "winningPlan" : {   # No SORT stage!!!
        "inputStage" : {
            "stage" : "IXSCAN",
            "keyPattern" : {    # Used our new index!!!
                "class_id" : 1,     # Class is very selective
                "final_grade" : 1,  # Walk index keys in order to get sorted result set
                "student_id" : 1    # Match student_id: {$gt: 500000} range constraint
            }
    "executionStats" : {
        "executionTimeMillis" : 27  # Super fast!!!

equality field: field on which queries will perform an equality test
sort field: field on which queries will specify a sort
range field: field on which queries perform a range test

Equality fields before range fields
Sort fields before range fields
Equality fields before sort fields

> db.students.find({students_id:10000})
{  "_id" : ObjectId(""), "student_id" : 10000, "scores" : [ { "type" : "exam", "score" : 23.69 }, { "type" : "quiz", "score" :
Sun Nov 18 [conn1] query school.students query: { student_id: 10000.0 } ntoreturn:0 nscanned:10000000 ... 4241ms  # took 4 seconds!!!

$ mongod -dbpath /usr/local/var/mongodb --profile 1 --slowms 2
> db.students.find({student_id:10000})  # <--- Slow query!!
> db.system.profile.find().pretty()
    "ns" : "school.students",
    "query" : {
        "student_id" : 10000
    }
    "nscanned" : 10000000,  # <-- Too many!!!
    "nreturned: 1,
    "millis" : 4231,  # <-- Took too long!!! (4 seconds)
> db.system.profile.find({ns:/school.students/}).sort({ts:1}).pretty()
> db.system.profile.find({millis:{$gt:1}}).sort({ts:1}).pretty()  # sort by timestamp

> db.getProfilingLevel()
1
> db.getProfilingStatus()
{"was" : 1, "slowms" : 2 }
> db.setProfilingStatus()
# Set the profiling level:
> db.setProfilingLevel(1,4) # <-- profile queries taking 4 seconds or longer
{ "was" : 1, "slowms" : 2, "ok" : 1 }
# Turn off profiling:
> db.setProfilingLevel(0)
{ "was" : 1, "slowms" : 2, "ok" : 1 }
> db.getProfilingStatus()
{"was" : 0, "slowms" : 4 }
> db.getProfilingLevel()
0

db.system.profile.find({millis:{$gt:1000}}).sort({ts:-1}).pretty()

mongotop provides a method to track the amount of time a MongoDB instance spends reading and writing data. mongotop provides statistics on a per-collection level. By default, mongotop returns values every second.

$ mongotop 3
$ mongo --port 27018
> use school
> db.students.count();
$ mongostat --port 27018
insert query update delete

The getmore column concerns the number of requests per time interval to get additional data from a cursor
the faults column appears only in the mmapv1 output

• db.products.find( { 'brand' : "GE" } )
• db.products.find( { 'brand' : "GE" } ).sort( { price : 1 } )
• db.products.find( { $and : [ { price : { $gt : 30 } },{ price : { $lt : 50 } } ] } ).sort( { brand : 1 } )
• db.products.find( { brand : 'GE' } ).sort( { category : 1, brand : -1 } )

• db.products.find( { 'brand' : "GE" } )
• db.products.find( { 'brand' : "GE" } ).sort( { price : 1 } )
• db.products.find( { $and : [ { price : { $gt : 30 } },{ price : { $lt : 50 } } ] } ).sort( { brand : 1 } )
• db.products.find( { brand : 'GE' } ).sort( { category : 1, brand : -1 } )

The following 2 combinations are wrong:
"$and" & "category : 1, brand : -1"
"price : 1" & "category : 1, brand : -1"
I now realize that the "category : 1, brand : -1" is wrong because either all or none of the keys must have reverse sort direction.

Correct way to solve:

> db.createCollection("products")
> db.products.insert(
   [
     { item: "digital camera", brand: "GE", price: 119.95, category: 'electronics' },
     { item: "Walkman", brand: "Sony", price: 99.95, category: 'electronics' },
     { item: "iPod", brand: "Apple", price: 399.95, category: 'electronics' },
     { item: "Civic", brand: "Honda", price: 10000, category: 'car' },
     { item: "micorave", brand: "GE", price: 139.95, category: 'appliance' }
   ]
)
BulkWriteResult({
	"writeErrors" : [ ],
	"writeConcernErrors" : [ ],
	"nInserted" : 5,
	"nUpserted" : 0,
	"nMatched" : 0,
	"nModified" : 0,
	"nRemoved" : 0,
	"upserted" : [ ]
})
> db.products.createIndex({	"price" : -1 });
> db.products.createIndex({ "category" : 1, "brand" : 1 });
> db.products.explain().find( { 'brand' : "GE" } );
> db.products.explain().find( { 'brand' : "GE" } ).sort( { price : 1 } );
			"inputStage" : {
				"stage" : "IXSCAN",
				"keyPattern" : {
					"price" : -1
				},
> db.products.explain().find( { $and : [ { price : { $gt : 30 } },{ price : { $lt : 50 } } ] } ).sort( { brand : 1 } );
						"inputStage" : {
							"stage" : "IXSCAN",
							"keyPattern" : {
								"price" : -1
							},
> db.products.explain().find( { brand : 'GE' } ).sort( { category : 1, brand : -1 } )
				"inputStage" : {
					"stage" : "COLLSCAN",
> db.students.getIndexes();
> db.students.dropIndex({student_id:1});

db.products.find( { 'brand' : "GE" } ).sort( { price : 1 } )
db.products.find( { $and : [ { price : { $gt : 30 } },{ price : { $lt : 50 } } ] } ).sort( { brand : 1 } )

• The query uses an index to determine the order in which to return result documents.
• The query examines 251120 documents.

$ mongoimport -d m101 -c profile < sysprofile.json
connected to: localhost
imported 1515 documents
> use m101
> db.profile.find({ns:/school2.students/}).sort({millis:-1}).limit(1).pretty()

Aggregation pipelines are similar to Linux shell pipelines.
The task a pipeline stage performs is tunable.
Pipelines use a MongoDB collection as input.
You may include a particular type of stage in an aggregation pipeline multiple times.
An individual stages within a pipeline performs a single task.

> db.companies.aggregate([
    { $match : { founded_year: 2004 } },
    { $sort: { name: 1 } },
    { $skip: 10 },
    { $limit: 5 },
    { $project: {
        _id: 0,
        name: 1
    }  }
])

• You should try to include $match stages as early as possible in your pipeline.
• To filter documents using a $match stage, we use the same syntax for constructing query documents (filters) as we do for find().

> db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock" } }
    ,
    { $project: {
        _id: 0,
        name: 1,
        ipo: "$ipo.pub_year",
        valuation: "$ipo.valuation_amount",
        funders: "$funding_rounds.investments.financial_org.permalink"
    } }
]).pretty()
> db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock } }
    ,
    { $projet: {
        _id: 0,
        name: 1,
        founded: {
            year: "$founded_year",
            month: "$founded_month",
            day: "$found_day"
        }
    } }
]).pretty()

db.companies.aggregate([
    { $match: {"name": "Facebook" } }
    ,
    { $project: {
        _id: 0,
        name: 1,
        people: "$relationships.person.last_name"
    } }
])

db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock" } },
    { $project: {
        _id: 0,
        name: 1,
        amount: "$funding_rounds.raised_amount",
        year: "$funding_rounds.funded_year"
    } }
])


// unwind
db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock" } },
    { $unwind: "$funding_rounds" },
    { $project: {
        _id: 0,
        name: 1,
        amount: "$funding_rounds.raised_amount",
        year: "$funding_rounds.funded_year"
    } }
])



// Add funder to output documents.
db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock" } },
    { $unwind: "$funding_rounds" },
    { $project: {
        _id: 0,
        name: 1,
        funder: "$funding_rounds.investments.financial_org.permalink",
        amount: "$funding_rounds.raised_amount",
        year: "$funding_rounds.funded_year"
    } }
])

db.companies.aggregate([
    { $match: {"funding_rounds.investments.financial_org.permalink": "greylock" } },
    { $project: {
        _id: 0,
        name: 1,
        founded_year: 1,
        rounds: { $filter: {
            input: "$funding_rounds",
            as: "round",
            cond: { $gte: ["$$round.raised_amount", 100000000] } } }
    } },
    { $match: {"rounds.investments.financial_org.permalink": "greylock" } },
]).pretty()





db.companies.aggregate([
    { $match: { "founded_year": 2010 } },
    { $project: {
        _id: 0,
        name: 1,
        founded_year: 1,
        first_round: { $arrayElemAt: [ "$funding_rounds", 0 ] },
        last_round: { $arrayElemAt: [ "$funding_rounds", -1 ] }
    } }
]).pretty()


db.companies.aggregate([
    { $match: { "founded_year": 2010 } },
    { $project: {
        _id: 0,
        name: 1,
        founded_year: 1,
        first_round: { $slice: [ "$funding_rounds", 1 ] },
        last_round: { $slice: [ "$funding_rounds", -1 ] }
    } }
]).pretty()

db.companies.aggregate([
    { $match: { "founded_year": 2010 } },
    { $project: {
        _id: 0,
        name: 1,
        founded_year: 1,
        early_rounds: { $slice: [ "$funding_rounds", 1, 3 ] }
    } }
]).pretty()

db.companies.aggregate([
    { $match: { "founded_year": 2004 } },
    { $project: {
        _id: 0,
        name: 1,
        founded_year: 1,
        total_rounds: { $size: "$funding_rounds" }
    } }
]).pretty()

db.companies.aggregate([
    { $match: { "funding_rounds": { $exists: true, $ne: [ ]} } },
    { $project: {
        _id: 0,
        name: 1,
        largest_round: { $max: "$funding_rounds.raised_amount" }
    } }
])


db.companies.aggregate([
    { $match: { "funding_rounds": { $exists: true, $ne: [ ]} } },
    { $project: {
        _id: 0,
        name: 1,
        total_funding: { $sum: "$funding_rounds.raised_amount" }
    } }
])


db.companies.aggregate([
    { $group: {
        _id: { founded_year: "$founded_year" },
        average_number_of_employees: { $avg: "$number_of_employees" }
    } },
    { $sort: { average_number_of_employees: -1 } }

])

db.companies.aggregate([
    { $group: {
        _id: { founded_year: "$founded_year" },
        average_number_of_employees: { $avg: "$number_of_employees" }
    } },
    { $sort: { average_number_of_employees: -1 } }

])

db.companies.aggregate( [
    { $match : { founded_year : 2001 } },
    { $project : { _id: 0, name : 1, number_of_employees: 1 } },
    { $sort : { number_of_employees : -1 } }
] )


db.companies.aggregate( [
    { $match: { "relationships.person": { $ne: null } } },
    { $project: { relationships: 1, _id: 0 } },
    { $unwind: "$relationships" },
    { $group: {
        _id: "$relationships.person",
        count: { $sum: 1 }
    } },
    { $sort: { count: -1 } }
] )

db.companies.aggregate([
    { $match: { founded_year: { $gte: 2010 } } },
    { $group: {
        _id: { founded_year: "$founded_year"},
        companies: { $push: "$name" }
    } },
    { $sort: { "_id.founded_year": 1 } }
]).pretty()


db.companies.aggregate([
    { $match: { founded_year: { $gte: 2010 } } },
    { $group: {
        _id: "$founded_year",
        companies: { $push: "$name" }
    } },
    { $sort: { "_id": 1 } }
]).pretty()



db.companies.aggregate([
    { $match: { founded_year: { $gte: 2010 } } },
    { $group: {
        _id: { founded_year: "$founded_year", category_code: "$category_code" },
        companies: { $push: "$name" }
    } },
    { $sort: { "_id.founded_year": 1 } }
]).pretty()


db.companies.aggregate([
    { $group: {
        _id: { ipo_year: "$ipo.pub_year" },
        companies: { $push: "$name" }
    } },
    { $sort: { "_id.ipo_year": 1 } }
]).pretty()


db.companies.aggregate( [
    { $match: { "relationships.person": { $ne: null } } },
    { $project: { relationships: 1, _id: 0 } },
    { $unwind: "$relationships" },
    { $group: {
        _id: "$relationships.person",
        count: { $sum: 1 }
    } },
    { $sort: { count: -1 } }
] )

db.companies.aggregate([
    { $match: { funding_rounds: { $ne: [ ] } } },
    { $unwind: "$funding_rounds" },
    { $sort: { "funding_rounds.funded_year": 1,
               "funding_rounds.funded_month": 1,
               "funding_rounds.funded_day": 1 } },
    { $group: {
        _id: { company: "$name" },
        funding: {
            $push: {
                amount: "$funding_rounds.raised_amount",
                year: "$funding_rounds.funded_year"
            } }
    } },
] ).pretty()



db.companies.aggregate([
    { $match: { funding_rounds: { $exists: true, $ne: [ ] } } },
    { $unwind: "$funding_rounds" },
    { $sort: { "funding_rounds.funded_year": 1,
               "funding_rounds.funded_month": 1,
               "funding_rounds.funded_day": 1 } },
    { $group: {
        _id: { company: "$name" },
        first_round: { $first: "$funding_rounds" },
        last_round: { $last: "$funding_rounds" },
        num_rounds: { $sum: 1 },
        total_raised: { $sum: "$funding_rounds.raised_amount" }
    } },
    { $project: {
        _id: 0,
        company: "$_id.company",
        first_round: {
            amount: "$first_round.raised_amount",
            article: "$first_round.source_url",
            year: "$first_round.funded_year"
        },
        last_round: {
            amount: "$last_round.raised_amount",
            article: "$last_round.source_url",
            year: "$last_round.funded_year"
        },
        num_rounds: 1,
        total_raised: 1,
    } },
    { $sort: { total_raised: -1 } }
] ).pretty()





db.companies.find({ name: "Fox Interactive Media" })

db.companies.aggregate( [
{ $match: { "relationships.person": { $ne: null } } },
{ $project: { relationships: 1, _id: 0 } },
{ $unwind: "$relationships" },
{ $group: {
_id: "$relationships.person",
count: { $sum: 1 }
} },
{ $sort: { count: -1 } }
] )

• 23 (23 is the total number of relationships Eric Di Benedetto is found in)

db.companies.aggregate( [
    { $match: { "relationships.person": { $ne: null } } },
    { $project: { name: 1, relationships: 1, _id: 0 } },
    { $unwind: "$relationships" },
    { $group: {
    	_id: { founded_year: "$relationships.person", name: "$name" },
        count: { $sum: 1 }
    } },
    { $match: { "_id.founded_year.permalink": "eric-di-benedetto" } },
    { $group: {
    	_id: { this_dude: "$_id.founded_year.permalink"},
        count: { $sum: 1 }
    } }
] )

$ mongoimport -d test -c grades --drop grades.json
2016-02-10T12:28:32.102-0600	connected to: localhost
2016-02-10T12:28:32.103-0600	dropping: test.grades
2016-02-10T12:28:32.172-0600	imported 280 documents
> use test
> db.grades.aggregate([
    { $unwind: "$scores" },
    { $match: {"scores.type": { $ne: "quiz" } } },
    { $group: {
        _id: "$class_id",
        average: { $avg: "$scores.score" }
    } },
    { $sort: { "average": -1 } }
    ]).pretty()
{ "_id" : 1, "average" : 64.89462395242741 }
{ "_id" : 9, "average" : 58.32770509408073 }
{ "_id" : 5, "average" : 56.80408386154663 }
        

1. $match (on the year)
2. $project (including an array size)
3. $match (on the size)
4. $project (calculating the avg)
5. $sort (by average amount raised)
6. $limit (to 1)

db.companies.aggregate( [
    { $match : { founded_year : 2004 } },
    {
        $project: {
            name: 1,
            raised_amount: "$funding_rounds.raised_amount",
            numberOfFundingRounds: { $size: "$funding_rounds" },
        }
    },
   { $match : { numberOfFundingRounds : {$gt: 4} } },
   {
        $project: {
        name: 1,
        raisedAvg: {$avg: "$raised_amount" }
                }
    },
          { $sort: { raisedAvg: 1 } },
    { $limit: 1 }
   ]).pretty()
{
	"_id" : ObjectId("52cdef7d4bab8bd675298d6e"),
	"name" : "Nimbit",
	"raisedAvg" : 1085127.2
}
        

Provided you assume that the disk is persistent, what are the w and j settings required to guarantee that an insert or update has been written all the way to disk.

• The network TCP connection between the application and the server was reset after the server received a write but before a response could be sent.
• The MongoDB server terminates between receiving the write and responding to it.
• The network fails between the time of the write and the time the client receives a response to the write.

• Regular replica set members
• Hidden Members
• Arbiters

$ more create_replica_set.sh
mkdir -p /data/rs1/ /data/rs2 /data/rs3
mongo --replSet rs1 --logpath "1.log" --dbpath /path/rs1 --port 27017
mongo --replSet rs1 --logpath "2.log" --dbpath /path/rs1 --port 27018
mongo --replSet rs1 --logpath "3.log" --dbpath /path/rs1 --port 27019
$ bash < create_replica_set.sh
$ more 1.log
$ more init_replica.js
$ config = { _id: "rs1", members:[
            { _id : 0, host : "Andrews-iMac.local:27017", priority:0, slaveDelay: 5 },
            { _id : 1, host : "Andrews-iMac.local:27018" },
            { _id : 2, host : "Andrews-iMac.local:27019"} ]
}

rs.initiate(config)
rs.status()
$ mongo --port 27018
bye
$ mongo --port 27018 < init_replica.js
bye
$ mongo --port 27018
rs1:PRIMARY> rs.status()
bye
$ mongo --port 27017
rs1:SECONDARY>
$ mongo --port 27019
rs1:SECONDARY> db.people.find()
error: { "$err" : "not master and slaveOk=false", "code" : 13435 }
rs1:SECONDARY> rs.slaveOk()

$ ps -ef | grep mongod
> use local
> db.oplog.rs.find().pretty()
$ ps -ef | grep mongod
60491  --port 27017 --smallfiles --fork
60494  --port 27018 --smallfiles --fork
> rs.staus();
{
            "_id" : 1,
            "name" : "localhost:27018",
            "stateStr" : "(not reachable/healthy)",
            "uptime : 0,
}
        

• Replication supports mixed-mode storage engines. For examples, a mmapv1 primary and wiredTiger secondary.
• A copy of the oplog is kept on both the primary and secondary servers.
• The oplog is implemented as a capped collection.
• You can only write to a primary node
• You cannot read from a secondary by default.

$ mongo localhost:30001
> rs.initiate()
> rs.add("education.local:30002")
> rs.add("education.local:30003")
> rs.status()
$ vim app.js
var MongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://localhost:30001,localhost:30002,localhost:30003/course", function(err, db) {
    if (err) throw err;

    db.collection("repl").insert({ 'x' : 1 }, function(err, doc) {
        if (err) throw err;

        db.collection("repl").findOne({ 'x' : 1 }, function(err, doc) {
            if (err) throw err;

            console.log(doc);
            db.close();
        });
    });
});

var MongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://localhost:30001,localhost:30002,localhost:30003/course", function(err, db) {
    if (err) throw err;

    var documentNumber = 0;
    function insertDocument() {

        db.collection("repl").insert({ 'documentNumber' : documentNumber++ }, function(err, doc) {
            if (err) throw err;
            console.log(doc);
        });

        console.log("Dispatched insert");
        setTimeout(insertDocument, 1000);
    }

    insertDocument();
});

db.collection('foo').insert({x:1}, callback);

        w How many nodes you wait for before you move on when doing an INSERT
        w=2 Wait for primary & secondary nodes to acknowledge write.
        j=1 Wait for primary to write to disk
        WTimeout = how long to wait for write to be acknowledged by secondaries.
        Places to configure:
        Set on a connection.
        Set on collection.
        Set on replica set.

            w=majority (wait for majority of nodes to replicate/avoid rollbacks)

> rs.conf()
"members" : [
     {
            "_id" : 0,
            "host" : "Andrews-iMac.local:27017",
            "priority" : 0,
            "slaveDelay" : 5
     }
        

• Primary (default)
• Primary Preferred
• Secondaries
• Secondaries Prefered
• Nearest (ping time, tag set (data center))

You can configure your applications via the drivers to read from secondary nodes within a replica set. What are the reasons that you might not want to do that? Check all that apply.

• You may not read what you previously wrote to MongoDB on a secondary because it will lag behind by some amount.
• If the secondary hardware has insufficient memory to keep the read working set in memory, directing reads to it will likely slow it down.
• If your write traffic is great enough, and your secondary is less powerful than the primary, you may overwhelm the secondary, which must process all the writes as well as the reads. Replication lag can result.

> db.students.explain().find({}).limit(10)
    "stage" : "SHARD_MERGE",
        "shardName" : "s0",
        "shardName" : "s1",
        "shardName" : "s2",
> db.students.explain().find({student_id:1000}).limit(10)
    "stage" : "SINGLE_SHARD",
        "shardName" : "s0",
            "inputStage" : {
                "student_index"
            }

• Every document needs to include the shard key
• Shard key is immutable
• Need index that starts with the shard key
• When doing an update, shard key needs to be specified or multi-key = true
• If query lacks shard key => scatter gather
• No unique key unless it starts with the shard key

• Choosing posttime as the shard key will cause hotspotting as time progresses.
• Choosing username as the shard key will distribute posts to the wall well across the shards.
• Choosing visible_to (multikey index) as a shard key is illegal.

• The oplog utilizes a capped collection.
• The minimum sensible number of voting nodes to a replica set is three.
• By default, using the MongoClient connection class, w=1 and j=0
• MongoDB replication is asynchronous.

• There must be an index on the collection that starts with the shard key.
• MongoDB cannot enforce unique indexes on a sharded collection other than the shard key itself or indexes preferred by the shard key.
• Any update that does not contain the shard key will be sent to all shards.

			{ "student_id" : { "$minKey" : 1 } } -->> { "student_id" : 0 } on : s2 Timestamp(3, 0)
			{ "student_id" : 0 } -->> { "student_id" : 2 } on : s0 Timestamp(3, 1)
			{ "student_id" : 2 } -->> { "student_id" : 3497 } on : s1 Timestamp(3, 2)
			{ "student_id" : 3497 } -->> { "student_id" : 7778 } on : s1 Timestamp(3, 3)
			{ "student_id" : 7778 } -->> { "student_id" : { "$maxKey" : 1 } } on : s1 Timestamp(3, 4)

# Create three directories for the three mongod processes.
$ mkdir -p /data/rs1 /data/rs2 /data/rs3
# Now start three mongo instances
$ mongod --replSet m101 --logpath "1.log" --dbpath /data/rs1 --port 27017 --smallfiles --oplogSize 64 --fork
$ mongod --replSet m101 --logpath "2.log" --dbpath /data/rs2 --port 27018 --smallfiles --oplogSize 64 --fork
$ mongod --replSet m101 --logpath "3.log" --dbpath /data/rs3 --port 27019 --smallfiles --oplogSize 64 --fork
# Now connect to a mongo shell and make sure it comes up
$ mongo --port 27017
# Now create the replica set.
> config = { _id: "m101", members:[
          { _id : 0, host : "localhost:27017"},
          { _id : 1, host : "localhost:27018"},
          { _id : 2, host : "localhost:27019"} ]
         };
> rs.initiate(config);
# see the state of replication
> rs.status()

Introduction

Getting Started

mongo some-host:30000/myDB

$ mongo --nodb
> conn = new Mongo("some-host:30000") connection to some-host:30000
> db = conn.getDB("myDB")
myDB

mongo script1.js script2.js script3.js

mongo --quiet server-1:30000/foo script1.js script2.js script3.js

> load("script1.js")
I am script1.js
# JavaScript equivalents to shell helpers
Helper              Equivalent
use foo             db.getSisterDB("foo")
show dbs            db.getMongo().getDBs()
show collections    db.getCollectionNames()

prompt = function() {
if (typeof db == 'undefined') {
return '(nodb)> '; }
        // Check the last db operation
try { db.runCommand({getLastError:1});
}
catch (e) {
print(e); }
return db+"> "; };

Welcome to Node.js

var stream = fs.createReadStream('./resource.json')
stream.on('data', function (chunk) {
  console.log(chunk)
})
stream.on('end', function () {
  console.log('finished')
})

Building a multiroom chat application

Building a multiroom chat application

Application overview

Application requirements and initial setup

Serving HTTP and WebSocket

Creating the application file structure

1. Launch Terminal
2. Create project directory

   $ mkdir chatroom
   $ cd chatroom/
   $ mkdir lib
   $ mkdir -p public/javascripts
   $ mkdir public/stylesheets
   

Specifying dependencies

1. Create and populate the package.json file

   $ touch package.json
   $ vi package.json
   i (Insert mode)
   (cut and paste the following:)
   {
     "name": "chatrooms",
     "version": "0.0.1",
     "description": "Minimalist multiroom chat server",
     "dependencies": {
       "socket.io": "~0.9.6",
       "mime": "~1.2.7"
     }
   }
   [Esc]       (turn off the Insert mode)
   :x[return]  (save & close)
   

Installing dependencies

$ npm install

Serving the application’s HTML, CSS, and client-side JavaScript

Creating a basic static file server

$ touch server.js
$ vi server.js
i (Insert mode)
var http  = require('http');
var fs    = require('fs');
var path  = require('path');
var mime  = require('mime');
var cache = {};
function send404(response) {
  response.writeHead(404, {'Content-Type': 'text/plain'});
  response.write('Error 404: resource not found.');
  response.end();
}
function sendFile(response, filePath, fileContents) {
  response.writeHead(
200,
{"content-type": mime.lookup(path.basename(filePath))} );
  response.end(fileContents);
}
function serveStatic(response, cache, absPath) {
  if (cache[absPath]) {
    sendFile(response, absPath, cache[absPath]);
  } else {
    fs.exists(absPath, function(exists) {
      if (exists) {
        fs.readFile(absPath, function(err, data) {
          if (err) {
            send404(response);
          } else {
            cache[absPath] = data;
            sendFile(response, absPath, data);
          }
        });
      } else {
        send404(response);
      }
    });
  }
}
var server = http.createServer(function(request, response) {
  var filePath = false;
  if (request.url == '/') {
    filePath = 'public/index.html';
  } else {
    filePath = 'public' + request.url;
  }
  var absPath = './' + filePath;
  serveStatic(response, cache, absPath);
});
server.listen(3000, function() {
          console.log("Server listening on port 3000.");
});
[Esc]       (turn off the Insert mode)
:w[return]  (save)
:x[return]  (save & close)
$ node server.js

Adding the HTML and CSS files

$ cd public
$ touch index.html
$ vi index.html
i (Insert mode)

        







  

  

  


  
  
  

    Chat commands:
    

      
Change nickname: /nick [username]
      
Join/create room: /join [room name]
    
  


 
 


[Esc]       (turn off the Insert mode)
:w[return]  (save)
:x[return]  (save & close)

$ cd stylesheets/
$ touch style.css
$ vi style.css
body {
  padding: 50px;
  font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
}
a {
  color: #00B7FF;
}
# content {
  width: 800px;
  margin-left: auto;
  margin-right: auto;
}
# room {
  background-color: #ddd;
  margin-bottom: 1em;
}
# messages {
   width: 690px;
   height: 300px;
   overflow: auto;
   background-color: #eee;
   margin-bottom: 1em;
   margin-right: 10px;
 }

Handling chat-related messaging using Socket.IO

Setting up the Socket.IO server

$ cd ../..
vi server.js
...
var chatServer = require('./lib/chat_server');
chatServer.listen(server);

$ cd lib
$ vi chat_server.js

Node programming fundamentals