back-end/social_network_analyzer/markets_7days_categorical_analysis.py
def findMostFrequentHashtags():

    print("Finding tweets with included hashtags from the Database, over the last 7 days.")
    print('Querying database and retrieving the data.')
    # computing the datetime now - 7 days ago
    sevenDaysAgo = datetime.datetime.utcnow() - datetime.timedelta(days=7)

    # Mongo Shell query
    # db.twitterQuery2.find({'entities.hashtags.text': {$exists : true}}, {'entities.hashtags.text': 1, '_id': 0}).limit(5)

    # creating query + projection for MongoDB
    query = {'$and': [{'entities.hashtags.text': {'$exists': 'true'}}, {'mongoDate': {'$gte': sevenDaysAgo}}]}
    projection = {'entities.hashtags.text': 1, '_id': 0}

    # running query
    cursor = []
    try:
        cursor = twitterOutput1.find(query, projection)
        # cursor = cursor.limit(20)

    except Exception as e:
        print("Unexpected error:", type(e), e)

    print("Finding used hashtags frequency..")
    # Listing countered hashtags coming from tweets for storing later the corresponding query in a CSV file
    countAllHashtags = Counter()
    hashtagsList = []
    for doc in cursor:
        hashtagsKey = doc['entities']['hashtags']
        for item in hashtagsKey:
            # print(item['text'])
            hashtagsList.append(('#' + item['text'].lower()))
            # countAllHashtags.update(item['text'])

    # print(hashtagsList)
    countAllHashtags.update(hashtagsList)

    print('Most 10 frequently used hashtags:')
    print(countAllHashtags.most_common(10))

    """
    CATEGORICAL ANALYSIS (BAR-PLOT) PANDAS SECTION
    """
    print('Starting data analysis with Pandas.')
    print('Creating data frame:')
    hash_freq = countAllHashtags.most_common(10)
    hash = pd.DataFrame(hash_freq)
    hash.set_index(0, inplace=True)
    print('Data frame:')
    print(hash.head())
    print('Writing CSV file..')
    hash.to_csv('/var/www/html/saint/twitterSNA-Aug17/hashtagsMarkets.csv')
    print('Writing Hashtags Categorical Analysis to CSV file completed!')

    CATEGORICAL ANALYSIS (BAR-PLOT) PANDAS SECTION
    
CATEGORICAL ANALYSIS (BAR-PLOT) PANDAS SECTION
CATEGORICAL ANALYSIS ( BAR - PLOT ) PANDAS SECTION
def findMostFrequentHashtags():
    print("Finding tweets with included hashtags from the Database, over the last 7 days.")
    print('Querying database and retrieving the data.')
    sevenDaysAgo = datetime.datetime.utcnow() - datetime.timedelta(days=7)
    query = {'$and': [{'entities.hashtags.text': {'$exists': 'true'}}, {'mongoDate': {'$gte': sevenDaysAgo}}]}
    projection = {'entities.hashtags.text': 1, '_id': 0}
    cursor = []
    try:
        cursor = twitterOutput1.find(query, projection)
    except Exception as e:
        print("Unexpected error:", type(e), e)
    print("Finding used hashtags frequency..")
    countAllHashtags = Counter()
    hashtagsList = []
    for doc in cursor:
        hashtagsKey = doc['entities']['hashtags']
        for item in hashtagsKey:
            hashtagsList.append(('#' + item['text'].lower()))
    countAllHashtags.update(hashtagsList)
    print('Most 10 frequently used hashtags:')
    print(countAllHashtags.most_common(10))
    print('Starting data analysis with Pandas.')
    print('Creating data frame:')
    hash_freq = countAllHashtags.most_common(10)
    hash = pd.DataFrame(hash_freq)
    hash.set_index(0, inplace=True)
    print('Data frame:')
    print(hash.head())
    print('Writing CSV file..')
    hash.to_csv('/var/www/html/saint/twitterSNA-Aug17/hashtagsMarkets.csv')
    print('Writing Hashtags Categorical Analysis to CSV file completed!')
def findMostFrequentHashtags ( ) : print ( "Finding tweets with included hashtags from the Database, over the last 7 days." ) print ( 'Querying database and retrieving the data.' ) sevenDaysAgo = datetime . datetime . utcnow ( ) - datetime . timedelta ( days = 7 ) query = { '$and' : [ { 'entities.hashtags.text' : { '$exists' : 'true' } } , { 'mongoDate' : { '$gte' : sevenDaysAgo } } ] } projection = { 'entities.hashtags.text' : 1 , '_id' : 0 } cursor = [ ] try : cursor = twitterOutput1 . find ( query , projection ) except Exception as e : print ( "Unexpected error:" , type ( e ) , e ) print ( "Finding used hashtags frequency.." ) countAllHashtags = Counter ( ) hashtagsList = [ ] for doc in cursor : hashtagsKey = doc [ 'entities' ] [ 'hashtags' ] for item in hashtagsKey : hashtagsList . append ( ( '#' + item [ 'text' ] . lower ( ) ) ) countAllHashtags . update ( hashtagsList ) print ( 'Most 10 frequently used hashtags:' ) print ( countAllHashtags . most_common ( 10 ) ) print ( 'Starting data analysis with Pandas.' ) print ( 'Creating data frame:' ) hash_freq = countAllHashtags . most_common ( 10 ) hash = pd . DataFrame ( hash_freq ) hash . set_index ( 0 , inplace = True ) print ( 'Data frame:' ) print ( hash . head ( ) ) print ( 'Writing CSV file..' ) hash . to_csv ( '/var/www/html/saint/twitterSNA-Aug17/hashtagsMarkets.csv' ) print ( 'Writing Hashtags Categorical Analysis to CSV file completed!' )
CATEGORICAL ANALYSIS (BAR-PLOT) PANDAS SECTION
CATEGORICAL ANALYSIS ( BAR - PLOT ) PANDAS SECTION
# computing the datetime now - 7 days ago # Mongo Shell query # db.twitterQuery2.find({'entities.hashtags.text': {$exists : true}}, {'entities.hashtags.text': 1, '_id': 0}).limit(5) # creating query + projection for MongoDB # running query # cursor = cursor.limit(20) # Listing countered hashtags coming from tweets for storing later the corresponding query in a CSV file # print(item['text']) # countAllHashtags.update(item['text']) # print(hashtagsList) """
    CATEGORICAL ANALYSIS (BAR-PLOT) PANDAS SECTION
    """