import pprint

def get_client():
    from pymongo import MongoClient
    return MongoClient('mongodb://localhost:27017/')

def get_collection():
    return get_client().examples.twitter

collection = get_collection()

def aggregate_and_show(collection, query, limit = True):
    _query = query[:]
    if limit:
        _query.append({"$limit": 5})
    result = collection.aggregate(_query)
    pprint.pprint(list(r for r in result))

query = [
    {"$group": {"_id": "$user.screen_name",
                "count": {"$sum": 1}}},
    {"$sort": {"count": -1}}
]
aggregate_and_show(collection, query)

"""
Explanation: Introduction
Will explore aggregation framework for some analysis and then explore how we could use it for data cleaning
Example of Aggregation Framework
Let's find out who tweeted the most
- group tweets by user
- count each user's tweets
- sort into descending order
- select user at top
End of explanation
"""


query = [
    {"$match": {"user.friends_count": {"$gt": 0},
                "user.followers_count": {"$gt": 0}}},
    {"$project": {"ratio": {"$divide": ["$user.followers_count", 
                                        "$user.friends_count"]},
                  "screen_name": "$user.screen_name"}},
    {"$sort": {"ratio": -1}}
]

aggregate_and_show(collection, query)

"""
Explanation: Aggregation Operators

$project - shape documents  e.g. select
$match - filtering
$skip - skip at start
$limit - limit after some
$unwind - for every field of the array field on which it is used it will create an instance of document containing the values of the field. This can be used for grouping

Match operator
Who has the highest followers to friend ratio?
End of explanation
"""


query = [
    {"$unwind": "$entities.user_mentions"},
    {"$group": {"_id": "$user.screen_name",
                "count": {"$sum": 1}}},
    {"$sort": {"count": -1}}
]

aggregate_and_show(collection, query)

"""
Explanation: For $match we use the same syntax that we use for read operations
Project operator

include fields from the original document
insert computed fields
rename fields
create fields that hold sub documents

Unwind operator

need to use array values somehow

Let's try and find who included the most user mentions
End of explanation
"""


#get unique hashtags by user
query = [
    {"$unwind": "$entities.hashtags"},
    {"$group": {"_id": "$user.screen_name",
                "unique_hashtags": {
                    "$addToSet": "$entities.hashtags.text"
                }}},
    {"$sort": {"_id": -1}}
]

aggregate_and_show(collection, query)

# find number of unique user mentions
query = [
    {"$unwind": "$entities.user_mentions"},
    {"$group": {
            "_id": "$user.screen_name",
            "mset": {
                "$addToSet": "$entities.user_mentions.screen_name"
            }
        }},
    {"$unwind": "$mset"},
    {"$group": {"_id": "$_id", "count": {"$sum": 1}}},
    {"$sort": {"count": -1}}
]

aggregate_and_show(collection, query)

"""
Explanation: group operators

$sum
$first
$last
$max
$min
$avg

array operators
- $push
- $addToSet
End of explanation
"""


