Spaces:
Running
Running
| import warnings | |
| warnings.filterwarnings("ignore") | |
| import json | |
| import requests | |
| import pandas as pd | |
| from lxml import etree | |
| from tqdm import tqdm | |
| from datetime import datetime | |
| from finnlp.data_sources.news._base import News_Downloader | |
| class SeekingAlpha_Date_Range(News_Downloader): | |
| def __init__(self, args = {}): | |
| super().__init__(args) | |
| def download_date_range_stock(self, start_date, end_date, stock = "AAPL", proxies = None): | |
| self.dataframe = pd.DataFrame() | |
| start_timestamp = int(datetime.strptime(start_date+'-13', "%Y-%m-%d-%H").timestamp()) | |
| end_timestamp = int(datetime.strptime(end_date+'-13', "%Y-%m-%d-%H").timestamp()) | |
| # Downloading First Page | |
| data, totalpages = self._gather_by_page(start_timestamp, end_timestamp, stock, 1, proxies) | |
| self.dataframe = pd.concat([self.dataframe, data]) | |
| # Downloading Other Pages | |
| with tqdm(total=totalpages, desc= "Downloading Titles") as bar: | |
| bar.update(1) | |
| for page in range(2, totalpages+1): | |
| data,_ = self._gather_by_page(start_timestamp, end_timestamp, stock, page, proxies) | |
| self.dataframe = pd.concat([self.dataframe, data]) | |
| bar.update(1) | |
| self.dataframe = self.dataframe.reset_index(drop = True) | |
| def _gather_by_page(self, start_timestamp, end_timestamp, stock, page = 1, proxies = None): | |
| url = f"https://seekingalpha.com/api/v3/symbols/{stock}/news?filter[since]={start_timestamp}&filter[until]={end_timestamp}&id={stock}&include=author%2CprimaryTickers%2CsecondaryTickers%2Csentiments&isMounting=true&page[size]=40&page[number]={page}" | |
| headers = { | |
| 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/113.0', | |
| 'Referer':f'https://seekingalpha.com/symbol/aapl/news?from=2009-12-31T16%3A00%3A00.000Z&to=2022-01-01T15%3A59%3A59.999Z' | |
| } | |
| response = requests.get(url, headers=headers, proxies=proxies) | |
| if response.status_code != 200: | |
| print(f"stock: {stock}, page: {page} went wrong!") | |
| return pd.DataFrame(), 1 | |
| else: | |
| res = json.loads(response.text) | |
| data = pd.DataFrame(res["data"]) | |
| # make new features | |
| new_columns = ["publishOn", "isLockedPro", "commentCount", "gettyImageUrl", "videoPreviewUrl", "themes", "title", "isPaywalled"] | |
| data[new_columns] = data.apply(lambda x:list(x.attributes.values()), axis = 1,result_type ="expand" ) | |
| new_columns = ["author", "sentiments", "primaryTickers", "secondaryTickers", "otherTags"] | |
| data[new_columns] = data.apply(lambda x:list(x.relationships.values()), axis = 1,result_type ="expand" ) | |
| # total pages | |
| totalpages = res["meta"]["page"]["totalPages"] | |
| return data, totalpages | |
| def obtain_content(self, parallel = False, proxies = None): | |
| if parallel: | |
| import os | |
| from pandarallel import pandarallel | |
| pandarallel.initialize(nb_workers=os.cpu_count()) | |
| self.dataframe['content'] = self.dataframe.parallel_apply(lambda x: self._obtain_content(x, proxies = proxies), axis = 1) | |
| else: | |
| self.dataframe['content'] = self.dataframe.apply(lambda x: self._obtain_content(x, proxies = proxies), axis = 1) | |
| def _obtain_content(self, x, proxies = None): | |
| url = x['links']['self'] | |
| url = f"https://seekingalpha.com{url}" | |
| headers = { | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.0' | |
| } | |
| res = requests.get(url, headers=headers, proxies=proxies) | |
| if res.status_code != 200: | |
| return '' | |
| else: | |
| resp = etree.HTML(res.text) | |
| resp = resp.xpath('//script[5]//text()') | |
| resp = resp[0].split('window.SSR_DATA = ')[1] | |
| resp = resp[:-1] | |
| resp = json.loads(resp) | |
| content = resp['article']['response']['data']['attributes']['content'] | |
| content = etree.HTML(content) | |
| content = content.xpath('//text()') | |
| content = [c if c!= ' ' else '\n' for c in content] | |
| content = ''.join(content) | |
| content = content.strip() | |
| return content | |