Hugging Face's logo

Spaces:
sigridveronica
/
ai-news-analyzer
Running

App Files Community
ai-news-analyzer
File size: 3,362 Bytes 
9df4cc0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
from finnlp.data_sources.social_media._base import Social_Media_Downloader

from tqdm import tqdm
from lxml import etree
import requests
import pandas as pd
import json
import base64

class Reddit_Streaming(Social_Media_Downloader):

    def __init__(self, args = {}):
        super().__init__(args)
        self.dataframe = pd.DataFrame()

    def download_streaming_all(self, rounds = 3):
        # Download the first page by url
        base_url = "https://www.reddit.com/r/wallstreetbets/new/"
        pbar = tqdm(total= rounds, desc= "Downloading by pages...")
        res = self._request_get(base_url)
        if res is None:
            raise ConnectionError
        
        # get the info from init page
        html = etree.HTML(res.text)
        init = html.xpath("//*[@id='data']/text()")[0]
        init = json.loads(init[14:][:-1])
        init = init["posts"]["models"]
        tmp_df = pd.DataFrame(init).T.reset_index(drop = True)
        self.dataframe = tmp_df
        init = [i for i in init if len(i)< 12]
        last_id = init[-1]
        last_id = self._encode_base64(last_id)
        
        pbar.update(1)

        # fetch other pages
        if rounds > 1:
            for _ in range(1,rounds):
                last_id = self._fatch_other_pages(last_id, pbar)

    def _fatch_other_pages(self, last_page, pbar):
        url = 'https://gql.reddit.com/'
        headers = {
            "referer":"https://www.reddit.com/",
            "authorization": "Bearer -twjFZkBAlpR8gZnZqsGHvz-G5c49PA",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
        }
        data = {
        "id": "02e3b6d0d0d7",
        "variables": {
            "name": "wallstreetbets",
            "includeIdentity": False,
            "adContext": {
            "layout": "CARD",
            "clientSignalSessionData": {
                "adsSeenCount": 4,
                "totalPostsSeenCount": 79,
                "sessionStartTime": "2023-04-07T15:32:13.933Z",
            }
            },
            "isFake": False,
            "includeAppliedFlair": False,
            "includeDevPlatformMetadata": True,
            "includeRecents": False,
            "includeTrending": False,
            "includeSubredditRankings": True,
            "includeSubredditChannels": False,
            "isAdHocMulti": False,
            "isAll": False,
            "isLoggedOutGatedOptedin": False,
            "isLoggedOutQuarantineOptedin": False,
            "isPopular": False,
            "recentPostIds": [],
            "subredditNames": [],
            "sort": "NEW",
            "pageSize": 25,
            "after": last_page
            }
        }
        response = self._request_post(url = url, headers= headers, json = data)
        data = json.loads(response.text)
        data = data["data"]["subredditInfoByName"]["elements"]["edges"]
        for d in data:
            if d["node"]["__typename"] == "SubredditPost":
                tmp = pd.DataFrame(d).T
                self.dataframe = pd.concat([self.dataframe, tmp])
                last_id = tmp.id.values[0]
        
        last_id = self._encode_base64(last_id)
        pbar.update(1)

        return last_id

    def _encode_base64(self,id):
        return base64.b64encode(id.encode('utf-8')).decode()