# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Break large collection into smaller collection based on week number
# ## 1. connect mongodb

# +
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)


import configparser 
config = configparser.ConfigParser()
config.read('config.ini')
ip = config['DEFAULT']['IP']
port = config['DEFAULT']['MongoDB-Port']

from pymongo import MongoClient
client = MongoClient(ip, int(port))
# -

#print(client.database_names())
db_twitter = client["Twitter"]
collections = db_twitter.collection_names()

# +
dic_collection={}
for i in collections:
    dic_collection[i] = "{:,}".format(db_twitter[i].find({}).count())
    
for key in sorted(dic_collection):
    print("%s: %s" % (key, dic_collection[key]))
# -

# ## 2. split twitter-other colection from 2019 week 4 to week 8

# +
# get timestamp_ms of 2019 from week 4 to week 8
list_2019_start = ["1548028800000","1548633600000","1549238400000","1549843200000","1550448000000"]
list_2019_end = ["1548633600000","1549238400000","1549843200000","1550448000000","1551052800000"]

for i in list_2019_start:
    index = list_2019_start.index(i)
    j= list_2019_end[index]
    w = int((int(i) - 1546214400000)/1000/604800 + 1)      # calculate week number
    
    # create pipeline to search record with timestamp_ms which is less than i
    pipeline = [ 
        {"$match": {'timestamp_ms':{'$gte':i,'$lt':j}}},
        {"$out": "2019_W" + str(w) + "_Twitter_Other"},
    ]
    
    # prevent emptying the collection if the method is already done before or some errors occur during the copying process
    try:
        if db_twitter['twitter-other'].find_one({'timestamp_ms':{'$lt':i}}) == None:
            print("No document is found")
            break
        db_twitter['twitter-other'].aggregate(pipeline)
    except:
        print("error")
        break
    
    print("finised 2019 week "+ str(w))

StarcoderdataJupyterScriptsDedupFiltered