# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Project Tweepler
# ### Project Team: <NAME>, James & Mandy
#
# #### Project Summary
# ##### Analysis of the trending twitter users globally to analyze trending topics, user behavior and sentiment.
# #### Tools / Imports Used
# -  Tweepler API to extract. Tweepler based on “ World” is based on the number of tweets per 20K tweets coming from the World in timespan of current to last 13 minutes.
# -  Jupyter Notebook / GitHub 
# -  Matplotlib/ Seaborn for graphing and animation.
# -  Beautiful Soup for scraping data Tweepler summary
# -  Re for scraping ISO 639 -1 language codes
# -  Vader Sentiment Analyzor
# -  WordCloud and Stopword to create wordcloud image of trending words
# -  Botometer
#
# #### Data Set and Quality Issues
# 500 tweets for top 10 trending users
# Location data not “usable” as it is a text field entered by users.
# 27.2% of data in tweets have multi language or language not supported by the Vader tool.
#
# #### Definitions
# Trending Users –twitter users mentioned by other users
# Language Group – “Other” - any language appearing less than 10 times in the data and is based on the tweet language.
#
# Tweeters - Users tweeting about the trending users.

import tweepy
import pandas as pd
import numpy as np
import requests
from bs4 import BeautifulSoup
import re
import matplotlib.pyplot as plt
import time
from datetime import datetime
import seaborn as sns
# Twitter API Keys
consumer_key = "rWYwPO5GujlB2DJnYxuiFXpv0"
consumer_secret = "<KEY>"
app_key = "891138909161242624-brnipgrui4Zl2ygfbVIrwc5OwEwRGIN"
app_secret = "<KEY>"
#Tweepy API Authentication
auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(app_key, app_secret)
#api = tweepy.API(auth)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

file = 'http://tweeplers.com/?cc=WORLD'
response = requests.get(file)

txt = response.text
top_names = re.findall("@(.*)<", txt)
top_names = [x.split('<')[0] for x in top_names][:10]
sentiment = []
compound_list = []
positive_list = []
negative_list = []
neutral_list = []
tweets = []
tweet_count = []

top_names

for user in top_names:
    num_tweets = 1
    #for status in tweepy.Cursor(api.user_timeline, id=user).items(100):
    for status in tweepy.Cursor(api.search, q=user).items(500):
        _status = status._json
        _status["top_user"] = user
        tweets.append(_status)
    print(user)


# +
def get_sentiment(tweet_text):
    results = analyzer.polarity_scores(tweet_text)
    return results

df = pd.DataFrame(tweets)
df['sentiment'] = df['text'].map(get_sentiment)
df['pos'] = df['sentiment'].map(lambda x: x.get('pos'))
df['neg'] = df['sentiment'].map(lambda x: x.get('neg'))
df['neu'] = df['sentiment'].map(lambda x: x.get('neu'))
df['comp'] = df['sentiment'].map(lambda x: x.get('compound'))
df['user_screen_name'] = df['user'].map(lambda x: x.get('screen_name'))
df['hashtags'] = df['entities'].map(lambda x: [j.get('text') for j in x.get('hashtags')]).apply(', '.join)
#df['hashtags'] = df['entities'].map(lambda x: [j.get('text') for j in x.get('hashtags')])
df['friends_count'] = df['user'].map(lambda x: x.get('friends_count')) 
df['tweet_count'] = df.index.values + 1
df['profile_url'] = df['user'].map(lambda x: x.get('profile_image_url'))
df['language'] = df['user'].map(lambda x: x.get('lang'))
df['location'] = df['user'].map(lambda x: x.get('location'))
df['profile_url'] = df['user'].map(lambda x: x.get('profile_image_url'))
df['statuses_count'] = df['user'].map(lambda x: x.get('statuses_count'))
df['user_created_at'] = df['user'].map(lambda x: x.get('created_at'))
df['verified'] = df['user'].map(lambda x: x.get('verified'))
df['user_name'] = df['user'].map(lambda x: x.get('name'))
df['description'] = df['user'].map(lambda x: x.get('description'))
df['followers_count'] = df['user'].map(lambda x: x.get('followers_count'))
df['listed'] = df['user'].map(lambda x: x.get('listed'))
df['favorites_count'] = df['user'].map(lambda x: x.get('favourites_count'))
df['notifications'] = df['user'].map(lambda x: x.get('notifications'))
df['following'] = df['user'].map(lambda x: x.get('following'))
#df['col'].apply(', '.join)
df
# -

df.to_csv("Twitter_Data.csv")

StarcoderdataJupyterScriptsDedupFiltered