# load json twitter data
twitter_json = r'data/twitter_01_20_17_to_3-2-18.json'
# Convert to pandas dataframe
tweet_data = pd.read_json(twitter_json)

"""
Explanation: Collect and Clean Twitter Data
The twitter data was obtained using the Trump Twitter Archive, the data is from 01/20/2017 - 03/02/2018 2:38 PM MST. I used the Federal Register's website to obtain all of the actions published by the Executive Office for the same time frame.
End of explanation
"""


# read the json data into a pandas dataframe
tweet_data = pd.read_json(twitter_json)
# set column 'created_at' to the index
tweet_data.set_index('created_at', drop=True, inplace= True)
# convert timestamp index to a datetime index
pd.to_datetime(tweet_data.index)

"""
Explanation: Using Pandas I will read the twitter json file, convert it to a dataframe, set the index to 'created at' as datetime objects, then write it to a csv
End of explanation
"""


# function to identify hash tags
def hash_tag(text):
    return re.findall(r'(#[^\s]+)', text) 
# function to identify @mentions
def at_tag(text):
    return re.findall(r'(@[A-Za-z_]+)[^s]', text)

# tokenize all the tweet's text
tweet_data['text_tokenized'] = tweet_data['text'].apply(lambda x: word_tokenize(x.lower()))
# apply hash tag function to text column
tweet_data['hash_tags'] = tweet_data['text'].apply(lambda x: hash_tag(x))
# apply at_tag function to text column
tweet_data['@_tags'] = tweet_data['text'].apply(lambda x: at_tag(x))

# pickle data
tweet_pickle_path = r'data/twitter_01_20_17_to_3-2-18.pickle'
tweet_data.to_pickle(tweet_pickle_path)

"""
Explanation: The next step is to add columns with tokenized text and identify twitter specific puncutiations like hashtags and @ mentions
End of explanation
"""


# Define the 2017 and 2018 url that contains all of the Executive Office of the President's published documents
executive_office_url_2017 = r'https://www.federalregister.gov/index/2017/executive-office-of-the-president' 
executive_office_url_2018 = r'https://www.federalregister.gov/index/2018/executive-office-of-the-president' 
# scrape all urls for pdf documents published in 2017 and 2018 by the U.S.A. Executive Office
pdf_urls= []
for url in [executive_office_url_2017,executive_office_url_2018]:
    response = requests.get(url)
    pattern = re.compile(r'https:.*\.pdf')
    pdfs = re.findall(pattern, response.text)
    pdf_urls.append(pdfs)
    

# writes all of the pdfs to the data folder
start = 'data/'
end = '.pdf'
num = 0
for i in range(0,(len(pdf_urls))):
    for url in pdf_urls[i]:
        ver = str(num)
        pdf_path = start + ver + end
        r = requests.get(url)
        file = open(pdf_path, 'wb')
        file.write(r.content)
        file.close()
        num = num + 1

"""
Explanation: Scrape Data from the Federal Register
This has already been done, and all of the pdfs published by the Executive Office of the U.S.A are in the data folder from 2017/01/20 - 2018/03/02
Don't execute this code unless you need more up-to-date information
End of explanation
"""


# function to convert pdf to text from stack overflow (https://stackoverflow.com/questions/26494211/extracting-text-from-a-pdf-file-using-pdfminer-in-python/44476759#44476759)
def convert_pdf_to_txt(path):
    rsrcmgr = PDFResourceManager()
    retstr = io.StringIO()
    codec = 'utf-8'
    laparams = LAParams()
    device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
    fp = open(path, 'rb')
    interpreter = PDFPageInterpreter(rsrcmgr, device)
    password = ""
    maxpages = 0
    caching = True
    pagenos = set()

    for page in PDFPage.get_pages(fp, pagenos, maxpages=maxpages,
                                  password=password,
                                  caching=caching,
                                  check_extractable=True):
        interpreter.process_page(page)

    text = retstr.getvalue()

    fp.close()
    device.close()
    retstr.close()
    return text
# finds the first time the name of a day appears in the txt, and returns that name

def find_day(word_generator):
    day_list = ['Monday,', 'Tuesday,', 'Wednesday,', 'Thursday,', 'Friday,', 'Saturday,', 'Sunday,']
    day_name_dict = {'Mon':'Monday,', 'Tue':'Tuesday,','Wed':'Wednesday,','Thu':'Thursday,','Fri':'Friday,','Sat':'Saturday,','Sun':'Sunday,'}
    day_name = []
    for val in word_generator:
        if val in day_list:
            num_position = txt.index(val)
            day_name.append(txt[num_position] + txt[num_position + 1] + txt[num_position +2])
            break
            
    return day_name_dict[day_name[0]]
# takes text and returns the first date in the document
def extract_date(txt):
    word_generator = (word for word in txt.split())
    day_name = find_day(word_generator)
    txt_start = int(txt.index(day_name))
    txt_end = txt_start + 40
    date_txt = txt[txt_start:txt_end].replace('\n','')
    cleaned_txt = re.findall('.* \d{4}', date_txt)
    date_list = cleaned_txt[0].split()
    clean_date_list = map(lambda x:x.strip(","), date_list)
    clean_date_string = ", ".join(clean_date_list)
    date_obj = datetime.strptime(clean_date_string, '%A, %B, %d, %Y')
    return date_obj


"""
Explanation: Create dataframe with the date the pdf was published and the text of each pdf
End of explanation
"""


start_path = r'data/'
end_path = '.pdf'
data_dict = defaultdict(list)
for i in range(0,270):
    file_path = start_path + str(i) + end_path
    txt = convert_pdf_to_txt(file_path)
    date_obj = extract_date(txt)
    data_dict[date_obj].append(txt)

"""
Explanation: Create a dictionary using DefaultDict where the date of publication is the key, and the text of the pdf is the value.
End of explanation
"""


tuple_lst = []
for k, v in data_dict.items():
    if v != None:
        for text in v:
            tuple_lst.append((k, text))
        

# create dataframe from list of tuples
fed_reg_dataframe = pd.DataFrame.from_records(tuple_lst, columns=['date','str_text'], index = 'date')

# tokenize all the pdf text
fed_reg_dataframe['token_text'] = fed_reg_dataframe['str_text'].apply(lambda x: word_tokenize(x.lower()))

# final dataframe
fed_reg_dataframe[fed_reg_dataframe.index > '2017-01-20']

"""
Explanation: Create a list of tuples, where the date is the first entry and the text of a pdf is the second entry, skipping over any values of None
End of explanation
"""


# pickle final data
fed_reg_data = r'data/fed_reg_data.pickle'
final_df.to_pickle(fed_reg_data)

"""
Explanation: Pickle the dataframe, so that you only need to process the text once
End of explanation
"""


