# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# ### Collect User Timeline
#
# This notebook is an example on how to download the full twitter history of a user id.
# Note: you will need an academic or professionnal token from twitter to do that.

# +
import requests
import json
import os
import time
import hashlib

from functools import wraps
from pathlib import Path
from enum import Enum


# -

def generate_token():
    with open(r"C:\Users\Orion\Documents\Projets\CERES\credentials_pro.json", 'r') as f:
        return f"Bearer {json.load(f)['token']}"


API_ROUTE = "https://api.twitter.com/2/"
STREAM_ROUTE = "tweets/search/stream"
RULES_ROUTE = "tweets/search/stream/rules"
FULL_SEARCH_ROUTE = "tweets/search/all"
ROOT_FOLDER = r"C:\Users\Orion\Documents\Projets\CERES\PMA\UsersTweets"
# maximum size of the collect in octets
MAX_SIZE = 10000000
TOKEN = generate_token()
params = {
    "tweet.fields": "public_metrics,referenced_tweets",
    "expansions": "author_id,in_reply_to_user_id,attachments.media_keys",
    "media.fields": "url",
    "user.fields": "id,verified"
}

s = requests.Session()
s.headers.update({"Authorization": TOKEN})


def get_full_user_tweets(user_id, resume_token=None):
    params_search = {
    "tweet.fields": "public_metrics,referenced_tweets,possibly_sensitive,created_at,source,reply_settings,withheld",
    "expansions": "author_id,in_reply_to_user_id,attachments.media_keys",
    "media.fields": "url,public_metrics,type,alt_text",
    "user.fields": "id,verified",
    "max_results": 500,
    "start_time": "2006-12-01T00:00:00Z",
    "end_time": "2021-09-26T00:00:00Z"
    }
    author_id = user_id
    query = 'from:'+ str(author_id)
    new_token = "" if not resume_token else resume_token
    i = 0
    total = 0
    while new_token is not None:
        if new_token != "":
            params_search['next_token'] = new_token
        res = s.get(API_ROUTE + FULL_SEARCH_ROUTE + '?query=' + query, params=params_search)
        data = res.json()
        count = data.get('meta', {}).get('result_count', 0)
        total += count
        print(f'on a récupéré {total} résultats')
        if  count == 0:
            print(f'aucun résultat pour cette requête, l"id est peut etre invalide: {user_id}')
            new_token = None
        with open(os.path.join(ROOT_FOLDER, str(author_id) + '-' + str(i) + '.json'), 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=4)
        new_token = data['meta'].get('next_token', None)
        if i % 10 == 0:
            print("5000 results collected, making a break")
            time.sleep(30)
        i += 1
    print('FINI')


get_full_user_tweets(19822406, resume_token="<PASSWORD>")

for id in [3336943089, 627503827, 493640854, 828290598599872512, 72489273, 2261320772, 2756277147]:
    get_full_user_tweets(id)

get_full_user_tweets(828290598599872515)

files = os.listdir(ROOT_FOLDER)
d = {}
for f in files:
    id_ = f.split('-')[0]
    if id_ not in d:
        d[id_] = []
    d[id_].append(f)
d.keys()


def tweets_to_csv(folder):
    head = "id;text;created_at;possibly_sensitive;reply_settings;source;retweet_count;reply_count;like_count;quote_count;referenced_tweets;images_url;in_reply_to_user_id;in_reply_to_user_name;in_reply_to_user_username"
    for key in d:
        with open(f'{key}.csv', 'w') as f:
            f.write(head)
        for i in range(len(d[key])):
            tweets = {}
            path = os.path.join(folder, f'{key}-{i}.json')
            print(path)
            with open(path, 'r', encoding='utf-8') as f:
                tweets = json.load(f)
            media = tweets['includes'].get('media', [])
            users = tweets['includes'].get('users', [])
            for tweet in tweets['data']:
                metrics = tweet['public_metrics']
                referenced_tweets = ' '.join([t['id'] for t in tweet.get('referenced_tweets', [])])
                media_keys = tweet.get('attachments', {}).get('media_keys', [])
                images_url = []
                if media_keys:
                    for medium in media:
                        if medium['media_key'] in media_keys and 'url' in medium:
                            images_url.append(medium['url'])
                in_reply_to_user_id = tweet.get('in_reply_to_user_id', None)
                in_reply_to = {}
                if in_reply_to_user_id:
                    for user in users:
                        if user['id'] == in_reply_to_user_id:
                            in_reply_to = user
                row = "\n"
                row += f"""{tweet['id']};"{tweet['text'].strip()}";{tweet['created_at']};""".replace('\n', ' ')
                row += f"{tweet['possibly_sensitive']};{tweet['reply_settings']};{tweet['source']};"
                row += f"{metrics['retweet_count']};{metrics['reply_count']};{metrics['like_count']};{metrics['quote_count']};"
                row += f"{referenced_tweets or 'NA'};"
                row += f"{' '.join(images_url) if images_url else 'NA'};"
                row += f"{in_reply_to_user_id or 'NA'};{in_reply_to.get('name', 'NA')};{in_reply_to.get('username', 'NA')}"
                with open(f'{key}.csv', 'a', encoding='utf-8') as f:
                    f.write(row)


tweets_to_csv(ROOT_FOLDER)

StarcoderdataJupyterScriptsDedupFiltered