Spaces:
Running
Running
| import warnings | |
| warnings.filterwarnings("ignore") | |
| from finnlp.data_sources.news._base import News_Downloader | |
| from tqdm import tqdm | |
| from lxml import etree | |
| import pandas as pd | |
| import requests | |
| import finnhub | |
| import time | |
| import json | |
| class Yahoo_Date_Range(News_Downloader): | |
| def __init__(self, args = {}): | |
| super().__init__(args) | |
| assert "token" in args.keys(), "Please input your finnhub token. Avaliable at https://finnhub.io/dashboard" | |
| self.finnhub_client = finnhub.Client(api_key=args["token"]) | |
| def download_date_range_stock(self, start_date, end_date, stock = "AAPL"): | |
| self.date_list = pd.date_range(start_date,end_date) | |
| self.dataframe = pd.DataFrame() | |
| days_each_time = 4 | |
| date_list = self.date_list | |
| # cal total lenth | |
| if len(date_list)%days_each_time == 0: | |
| total = len(date_list)//days_each_time | |
| else: | |
| total = len(date_list)//days_each_time+1 | |
| with tqdm(total=total, desc= "Downloading Titles") as bar: | |
| while len(date_list): | |
| tmp_date_list = date_list[:days_each_time] | |
| date_list = date_list[days_each_time:] | |
| tmp_start_date = tmp_date_list[0].strftime("%Y-%m-%d") | |
| tmp_end_date = tmp_date_list[-1].strftime("%Y-%m-%d") | |
| res = self._gather_one_part(tmp_start_date,tmp_end_date,stock = stock ) | |
| self.dataframe = pd.concat([self.dataframe,res]) | |
| bar.update(1) | |
| # res = self.finnhub_client.company_news(stock, _from=start_date, to=end_date) | |
| self.dataframe.datetime = pd.to_datetime(self.dataframe.datetime,unit = "s") | |
| self.dataframe = self.dataframe.reset_index(drop = True) | |
| def _gather_one_part(self, start_date, end_date, stock = "AAPL", delay = 1): | |
| res = self.finnhub_client.company_news(stock, _from=start_date, to=end_date) | |
| time.sleep(delay) | |
| return pd.DataFrame(res) | |
| def gather_content(self, delay = 0.01): | |
| pbar = tqdm(total = self.dataframe.shape[0], desc= "Gathering news contents") | |
| self.dataframe["content"] = self.dataframe.apply(lambda x:self._gather_content_apply(x, pbar, delay), axis = 1) | |
| def _gather_content_apply(self,x, pbar, delay = 0.01): | |
| time.sleep(delay) | |
| url = x.url | |
| source = x.source | |
| response = self._request_get(url = url) | |
| # response = self._request_get(url= url, headers= headers) | |
| pbar.update(1) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| try: | |
| # Yahoo Finance | |
| if source == "Yahoo": | |
| page = page.xpath("/html/body/div[3]/div[1]/div/main/div[1]/div/div/div/div/article/div/div/div/div/div/div[2]/div[4]") | |
| content = page[0].xpath(".//text()") | |
| content = "\n".join(content) | |
| return content | |
| # Reuters | |
| elif source == "Reuters": | |
| page = page.xpath("/html/body/div[1]/div[3]/div/main/article/div[1]/div[2]/div/div/div[2]") | |
| content = page[0].xpath(".//text()") | |
| content = "\n".join(content) | |
| return content | |
| # SeekingAlpha | |
| elif source == "SeekingAlpha": | |
| page = page.xpath("/html/body/div[2]/div/div[1]/main/div/div[2]/div/article/div/div/div[2]/div/section[1]/div/div/div") | |
| content = page[0].xpath(".//text()") | |
| content = "\n".join(content) | |
| return content | |
| # PennyStocks | |
| elif source == "PennyStocks": | |
| page = page.xpath("/html/body/div[3]/div/div[1]/div/div/div/main/article/div[2]/div[2]/div") | |
| content = page[0].xpath(".//text()") | |
| content = "\n".join(content) | |
| return content | |
| # MarketWatch | |
| elif source == "MarketWatch": | |
| page = page.xpath('//*[@id="js-article__body"]') | |
| content = page[0].xpath(".//text()") | |
| content = "".join(content) | |
| while " " in content: | |
| content = content.replace(" ", " ") | |
| while "\n \n"in content: | |
| content = content.replace("\n \n", " ") | |
| while "\n "in content: | |
| content = content.replace("\n ", " ") | |
| return content | |
| # Seeking Alpha | |
| elif source == "Seeking Alpha": | |
| # first get Seeking Alpha URL | |
| page = page.xpath('/html/body/div[5]/div[2]/section[1]/article[2]/div/div[2]/p/a/@href') | |
| url_new = page[0] | |
| response = self._request_get(url= url_new) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| content = page[0].xpath(".//text()") | |
| content = "\n".join(content) | |
| return content | |
| # Alliance News | |
| elif source == "Alliance News": | |
| page = page.xpath('//*[@id="comtext"]') | |
| content = page[0].xpath(".//text()") | |
| content = [c for c in content if not str(c).startswith("\r\n")] | |
| content = "\n".join(content) | |
| return content | |
| # Thefly.com | |
| elif source == "Thefly.com": | |
| page = page.xpath('/html/body/div[5]/div[2]/section[1]/article[2]/div/div[2]/p/a/@href') | |
| url_new = page[0] | |
| response = self._request_get(url= url_new, verify= False) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| page = page.xpath('/html/body/div[2]/div/div/div/div/div[2]/div[2]//text()') | |
| # content = page[0].xpath(".//text()") | |
| # content = [c for c in content if not str(c).startswith("\r\n")] | |
| content = "\n".join(page) | |
| content = content.replace("\r\n","") | |
| return content | |
| # TalkMarkets | |
| elif source == "TalkMarkets": | |
| return "Not supported yet" | |
| # CNBC | |
| elif source == "CNBC": | |
| page = page.xpath('/html/body/div[3]/div/div[1]/div[3]/div/div/div/div[3]/div[1]/div[2]/div[3]//text()') | |
| content = "\n".join(page) | |
| return content | |
| # GuruFocus | |
| elif source == "GuruFocus": | |
| page = page.xpath('/html/body/div[5]/div[2]/section[1]/article[2]/div/div[2]/p/a/@href') | |
| url_new = page[0] | |
| response = self._request_get(url= url_new) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| page = page.xpath('/html/body/div[1]/div/section/section/main/section/main/div[1]/div/div/div[1]/div[2]/div//text()') | |
| page_new = [] | |
| for c in page: | |
| while "\n" in c: | |
| c = c.replace("\n","") | |
| while " "in c: | |
| c = c.replace(" ","") | |
| page_new.append(c) | |
| content = "\n".join(page_new) | |
| return content | |
| # InvestorPlace | |
| elif source == "InvestorPlace": | |
| page = page.xpath('/html/body/div[5]/div[2]/section[1]/article[2]/div/div[2]/p/a/@href') | |
| url_new = page[0] | |
| response = self._request_get(url= url_new) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| page = page.xpath('//script[@type="application/ld+json"]')[1] | |
| content = page.xpath(".//text()") | |
| content = json.loads(content[0]) | |
| content = content["articleBody"] | |
| return content | |
| # TipRanks | |
| elif source == "TipRanks": | |
| page = page.xpath('/html/body/div[5]/div[2]/section[1]/article[2]/div/div[2]/p/a/@href') | |
| url_new = page[0] | |
| response = self._request_get(url= url_new) | |
| if response is None: | |
| return "Connection Error" | |
| else: | |
| page = etree.HTML(response.text) | |
| # /html/body/div[1]/div[2]/div[5]/div[2]/div[2]/div/div[6]/div/article/p[1]/p | |
| page = page.xpath('/html/body/div[1]/div[1]/div[4]/div[2]/div[2]/div[1]/div[6]//text()') | |
| # content = page[0].xpath('.//text()') | |
| page = [p.replace("\n","") for p in page] | |
| content = "".join(page) | |
| return content | |
| else: | |
| return "Not supported yet" | |
| except: | |
| return "Error" | |