Spaces:
Running
Running
| import warnings | |
| warnings.filterwarnings("ignore") | |
| import requests | |
| from lxml import etree | |
| from tqdm import tqdm | |
| import pandas as pd | |
| import json | |
| import time | |
| from finnlp.data_sources.news._base import News_Downloader | |
| # TODO: | |
| # 1. Contents | |
| class InvestorPlace_Streaming(News_Downloader): | |
| def __init__(self, args={}): | |
| super().__init__(args) | |
| self.dataframe = pd.DataFrame() | |
| def download_streaming_search(self, keyword = "apple", rounds = 3, delay = 0.5): | |
| url = 'https://investorplace.com/search/' | |
| headers = { | |
| "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" | |
| } | |
| print("Downloading ...", end = ' ') | |
| for page in range(rounds): | |
| params = { | |
| 'q': keyword, | |
| "pg": page, | |
| } | |
| res = requests.get(url = url, params=params, headers=headers) | |
| if res.status_code != 200: | |
| break | |
| res = etree.HTML(res.text) | |
| div_list = res.xpath("/html/body/main/section/div/div/div/div[2]/div[1]/div[1]/div") | |
| divs = [] | |
| for div in div_list: | |
| divs += div.xpath("./div") | |
| titles = [] | |
| times = [] | |
| authors = [] | |
| summaries = [] | |
| for div in divs: | |
| try: | |
| title = div.xpath('./h2/a//text()')[0] | |
| except: | |
| title = '' | |
| try: | |
| time_ = div.xpath('div/time//text()')[0].replace('\n','').replace('\t','') | |
| except: | |
| time_ = '' | |
| try: | |
| author = div.xpath('div/span/a/text()')[0].replace('\n','').replace('\t','') | |
| except: | |
| author = '' | |
| try: | |
| summary = div.xpath('p/text()')[0].replace('\n','').replace('\t','') | |
| except: | |
| summary = '' | |
| titles.append(title) | |
| times.append(time_) | |
| authors.append(author) | |
| summaries.append(summary) | |
| titles.append(title) | |
| tmp = pd.DataFrame([titles, times, authors, summaries]).T | |
| tmp.columns = ['title', 'time', 'author', 'summary'] | |
| self.dataframe = pd.concat([self.dataframe, tmp]) | |
| print(page, end = ' ') | |
| time.sleep(delay) | |