import warnings
warnings.filterwarnings("ignore")
import requests
from lxml import etree
from tqdm import tqdm
import pandas as pd
import json
import time
from finnlp.data_sources.news._base import News_Downloader

# TODO:
# 1. Contents
# 2. More pages

class TheFly_Streaming(News_Downloader):

    def __init__(self, args={}):
        super().__init__(args)
        self.dataframe = pd.DataFrame()

    def download_streaming_search(self, keyword = "AAPL",end_date = None, rounds = 3, delay = 0.5):
        # download first page
        self._download_first_page(keyword, delay = delay, end_date = end_date)
       
        # download the following pages
        # self._download_other_pages(keyword)
        print("Only support the first page now!")

    def _download_first_page(self, keyword = "AAPL", delay = 0.5, end_date = None):
        url = "https://thefly.com/news.php"
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
        }
        params = {
            'fecha': end_date,
            'market_stories': 'on',
            'hot_stocks_filter': 'on',
            'rumors_filter': 'on',
            'general_news_filter': 'on',
            'periodicals_filter': 'on',
            'earnings_filter': 'on',
            'technical_analysis_filter': 'on',
            'options_filter': 'on',
            'syndicates_filter': 'on',
            'onthefly': 'on',
            'insight_filter': 'on',
            'market_mover_filter': 'on',
            'e_inter_filter': 'on',
            'mid_wrap_filter': 'on',
            'sec_wrap_filter': 'on',
            'analyst_wrap_filter': 'on',
            'analyst_recommendations': 'on',
            'upgrade_filter': 'on',
            'downgrade_filter': 'on',
            'initiate_filter': 'on',
            'no_change_filter': 'on',
            'events': 'on',
            'symbol': keyword, 
        }
        res = requests.get(url = url, headers= headers, params = params, verify=False)
        if res.status_code != 200:
            print(f'Connection Error: {res.status_code}')
            return f'Connection Error: {res.status_code}'
        
        res = etree.HTML(res.text)
        tables = res.xpath("/html/body/div[2]/div/div/div[1]/table")[1:]
        titles = []
        stocks = []
        abstracts = []
        dates = []
        times = []
        for table in tables:
            trs = table.xpath("./tr")
            for tr in trs:
                title = tr.xpath("./td[2]/div[1]/a/span//text()")
                if len(title) > 0:
                    titles.append(' '.join(title))
                    stocks.append(' '.join(tr.xpath("./td[2]/div[1]/div/span/text()")))
                    abstracts.append(' '.join(tr.xpath("./td[2]/div[2]/dd/p[1]/text()")))
                    dates.append(' '.join(tr.xpath("./td[2]/div[1]/span[2]/small/span[3]/text()")))
                    times.append(' '.join(tr.xpath("./td[2]/div[1]/span[2]/small/span[3]/div/text()")))

        tmp = pd.DataFrame([titles, stocks, abstracts, dates, times]).T
        tmp.columns = ["title", "stock", "abstract", "date", "time"]
        self.dataframe = pd.concat([self.dataframe, tmp])

        time.sleep(delay)