import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline

plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (15, 3)
plt.rcParams['font.family'] = 'sans-serif'

pd.set_option('display.width', 5000) 
pd.set_option('display.max_columns', 60)

"""
Explanation: Google 'Image Box' analysis
First import all the libraries we want, together with some formatting options
End of explanation
"""


cols = ['requested_at', 'search_query', 'visible_link', 'rank' ,'image_path']

data = pd.read_csv('./image_box_data.csv', parse_dates=['requested_at'], usecols=cols)
data.head()

type(data.requested_at[0])

"""
Explanation: Read in Google Scraper search results table
End of explanation
"""


# Check the shelf has content. If nothing is printed out, run the following two scripts/cells
import shelve
db = shelve.open('db.shelve')
for k in db.keys():
    print(k)
db.close()

%run ../index.py --dataset ./clinton --shelve db.shelve

%run ../index.py --dataset ./trump --shelve db.shelve

"""
Explanation: Programatically identify unique images


Read in each image, calculate a unique hash based on the image data, a bit like a fingerprint, and store the hash along with the image filename in a shelf (a persistent dictionary). Many images were repeats, so there were many image filenames stored with the same hash key. 


The code is based on this blog post: https://realpython.com/blog/python/fingerprinting-images-for-near-duplicate-detection/ which I adapted to work with Python3
End of explanation
"""


import shelve

db = shelve.open('db.shelve')

for key in db.keys():   #  For every hash KEY
    for f in db[key]:   # for every file path name ITEM within the KEY
        for index, i in enumerate(data.image_path):      # For every Image path in each row of my DF
            if f in i:                                   # If the ITEM file path is also in the IMAGE PATH of my DF
                data.loc[index, 'image_hash'] = key      # Put the KEY into the 'image_hash' Column

data.to_csv("./hashedDF.csv", index=False)

db.close()

"""
Explanation: Add the hash to each row in our data dataframe we loaded above
End of explanation
"""


hashedDF = pd.read_csv('./hashedDF.csv')
hashedDF.head()

hashedDF['requested_at'] = pd.to_datetime(hashedDF['requested_at'])

type(hashedDF.requested_at[0])

len(hashedDF)

"""
Explanation: Open new dataframe with image hashes
End of explanation
"""


HC = hashedDF[hashedDF.search_query == 'hillary clinton']
HC.to_csv('HC_hashed.csv', index=False)

HC = pd.read_csv('HC_hashed.csv')

# What are the news sources, and how many times do they appear in the dataset?
HC.visible_link.value_counts()

# What are the hashes, and how many times does each one appear in the dataset?
HC.image_hash.value_counts()

print(len(HC.visible_link.unique()))
print(len(HC.image_hash.unique()))

HC.head()

print(type(HC.image_hash[0]))
print(type(HC['rank'][0]))
print(type(HC.requested_at[0]))

"""
Explanation: HILLARY CLINTON DATA
End of explanation
"""


HC_unique_images = HC.groupby('image_hash').first().reset_index()
HC_unique_images

"""
Explanation: Find an image file representative of each unique hash so we can look at each unique image
End of explanation
"""


from shutil import copyfile
import os

def select_images(df_series, src_dir, dest_dir):
    '''
    provide dataframe series with all the image file names, the directory containing the images, and directory 
    where the unique images should go.
    '''
    try:
        os.mkdir(dest_dir)
        for file in df_series:
            file = file.split('/')[-1] 
            copyfile(src_dir + file, dest_dir + file)
        
    except FileExistsError:
        pass

select_images(HC_unique_images.image_path,'clinton/', 'clinton_unique/' )

"""
Explanation: Collect unique images and put in separate directory
End of explanation
"""


DT = hashedDF[hashedDF.search_query == 'donald trump']
DT.to_csv('DT_hashed.csv', index=False)
DT = pd.read_csv('DT_hashed.csv')

DT.visible_link.value_counts()

DT.image_hash.value_counts()

"""
Explanation: DONALD TRUMP DATA
End of explanation
"""


DT_unique_images = DT.groupby('image_hash').first().reset_index()
DT_unique_images

"""
Explanation: Find an image file representative of each unique hash.
End of explanation
"""


select_images(DT_unique_images.image_path,'trump/', 'trump_unique/' )

"""
Explanation: Collect unique images and put in separate directory
End of explanation
"""


HC.visible_link.describe()

DT.visible_link.describe()

"""
Explanation: NEWS SOURCE INFORMATION
End of explanation
"""


allsides = pd.read_json('../BASELINE/allsides_data.json')
allsides.head()

HC_unique_news_sources = []

HC.visible_link.unique()

HC[HC.visible_link.isnull()]

def get_unique_news_sources(col, source_list):
    print(len(col.unique()))
    for i in col.unique():
        print(i)
        source_list.append(i.split('//')[1].split('/')[0])

get_unique_news_sources(HC.visible_link, HC_unique_news_sources)

HC_unique_news_sources

DT_unique_news_sources = []
get_unique_news_sources(DT.visible_link, DT_unique_news_sources)

DT_unique_news_sources

def get_url(col):
    col = col.split('//')[1]
    col = col.split('/')[0]
    return col

HC.loc[:, 'news_source_url'] = HC.visible_link.apply(get_url)
DT.loc[:, 'news_source_url'] = DT.visible_link.apply(get_url)

HC.head()

allsides.head()

def tag_bias_rating(candidate):
    
    candidate.loc['allsides_bias_rating'] = 999
    allsides = pd.read_json('../BASELINE/allsides_data.json')

    for i, valuei in enumerate(candidate.news_source_url):
        for j, valuej in enumerate(allsides.url):
                        
            if 'http' in valuej:
#                 print("Found an HTTP in ", valuej)
                valuej = valuej.split('//')[1]
#                 print(valuej)
            try:
                if valuei in valuej:          
                    print(valuei, valuej)

                    
                if allsides.loc[j, 'bias_rating'] == 71: # Left
                    candidate.loc[i, 'allsides_bias_rating'] = -2
                    
                elif allsides.loc[j, 'bias_rating'] == 72: # Lean left
                    candidate.loc[i, 'allsides_bias_rating'] = -1
                    
                elif allsides.loc[j, 'bias_rating'] == 73: # center
                    candidate.loc[i, 'allsides_bias_rating'] = 0
                    
                elif allsides.loc[j, 'bias_rating'] == 74: # lean right
                    candidate.loc[i, 'allsides_bias_rating'] = 1      
                    
                elif allsides.loc[j, 'bias_rating'] == 75: # Right
                    candidate.loc[i, 'allsides_bias_rating'] = 2                   
                   
                else:
                    candidate.loc[i, 'allsides_bias_rating'] = 999
                    
            except TypeError:
                continue

tag_bias_rating(HC)

tag_bias_rating(DT)

for i in allsides.url:
    if 'http' in i:
        print(i.split('//')[1])
    else:
        print(i)

"""
Explanation: Getting Political Leaning from Allsides from News Sources of all images in basline dataset
 
Allsides bias data was generously provided by Allsides
End of explanation
"""


facebook = pd.read_csv('../Facebook_study.csv')

facebook.head()

cols = ['p', 'avg_align']
facebook = pd.read_csv('../Facebook_study.csv', usecols=cols)
facebook.head()

def tag_facebookbias_rating(candidate):
    
    candidate['facebook_p'] = ''
    candidate['facebookbias_rating'] = 999
    
    count = 0
    for i, valuei in enumerate(candidate.visible_link):
        count += 1
        valuei = valuei.split('//')[1]
        valuei = valuei.split('/')[0]
        print(valuei, count)
        
        for j, valuej in enumerate(facebook.p):
            
            if valuej == valuei:                  
                print(valuei, valuej)
                candidate.loc[i, 'facebookbias_rating'] = facebook.loc[j, 'avg_align']
                candidate.loc[i, 'facebook_p'] = valuej

tag_facebookbias_rating(HC)

tag_facebookbias_rating(DT)

DT.facebookbias_rating[DT.facebookbias_rating < 3].plot.hist(alpha=0.5, bins=20, range=(-1,1), color='red')
HC.facebookbias_rating[HC.facebookbias_rating < 3].plot.hist(alpha=0.5, bins=20, range=(-1,1), color='blue')
plt.savefig('imagebox_facebookbias_hist.png')

DT.facebookbias_rating[DT.facebookbias_rating > 3].plot.hist(alpha=0.5, bins=10, range=(998,1000), color='red')
HC.facebookbias_rating[HC.facebookbias_rating > 3].plot.hist(alpha=0.5, bins=10, range=(998,1000), color='blue')
plt.savefig('imagebox_facebookbias_hist_unknowns.png')

HC.facebookbias_rating[HC.facebookbias_rating > 3].plot.hist()

DT.facebookbias_rating[DT.facebookbias_rating > 3].plot.hist()

HC.facebookbias_rating.value_counts()

DT.facebookbias_rating.value_counts()

"""
Explanation: Getting political leaning from a Facebook political bias ratings study
Data is available here
End of explanation
"""


def convert_facebookbias_toInts(col):
    
    if col >= 0.6 and col <= 1:
        return 2
    elif col >= 0.2 and col < 0.6:
        return 1
    elif col > -0.2 and col < 0.2:
        return 0
    elif col > -0.6 and col <= -0.2:
        return -1
    elif col <= -0.6:
        return -2
    elif col == 999:
        return 999
    else:
        return 999

HC['facebook_int'] = HC.facebookbias_rating.apply(convert_facebookbias_toInts)
DT['facebook_int'] = DT.facebookbias_rating.apply(convert_facebookbias_toInts)

HC.head()

HC.facebook_int.value_counts()

DT.facebook_int.value_counts()

"""
Explanation: Combining ratings from Allsides and Facebook, together with crowdsourced political bias ratings from Mondo Times and author's judgement
First, Facebook ratings were categoriesed into 2, 1, 0, -1, and -2 integers so as to be compatable with the Allsides data. Between-category thresholds were selected so as to make each bucket equal in float range of values.
End of explanation
"""


def combine_ratings(candidate):
    
    candidate['combine_rating'] = 'Not Rated'
    
    for i, valuei in enumerate(candidate.allsides_bias_rating):
        
        try:
            # STATEMENTS FOR IF BOTH RATINGS AGREE:
            # Both bias ratings say LEFT
            if (valuei < 0) and (candidate.loc[i, 'facebook_int'] < 0):
                print(valuei, candidate.loc[i, 'facebook_int'],  "Left")
                candidate.loc[i, 'combine_rating'] = "Left"
          
            # Both bias ratings say CENTER
            elif (valuei == 0.0) and (candidate.loc[i, 'facebook_int'] == 0):
                print(valuei, candidate.loc[i, 'facebook_int'],  "Center")
                candidate.loc[i, 'combine_rating'] = "Center"
        
            # Both bias ratings say RIGHT
            elif (0 < valuei < 3) and (0 < candidate.loc[i, 'facebook_int'] < 3):
                print(valuei, candidate.loc[i, 'facebook_int'], "Right")
                candidate.loc[i, 'combine_rating'] = "Right"
        
            # STATEMENTS FOR IF RATINGS ARE ONLY PRESENT IN ONE (ALLSIDES OR FACEBOOK STUDY)
            # Only one scale has a rating of LEFT, while the other has no entry 
            elif (valuei < 0 and candidate.loc[i, 'facebook_int'] == 999) or (valuei == 999 and candidate.loc[i, 'facebook_int'] < 0):
                print(valuei, candidate.loc[i, 'facebook_int'],  "Left")
                candidate.loc[i, 'combine_rating'] = "Left"
            
            # Only one scale has a rating of CENTER, while the other has no entry 
            elif (valuei == 0 and candidate.loc[i, 'facebook_int'] == 999) or (valuei  == 999 and candidate.loc[i, 'facebook_int'] == 0):
                print(valuei, candidate.loc[i, 'facebook_int'],  "Center")
                candidate.loc[i, 'combine_rating'] = "Center"
        
            # Only one scale has a rating of RIGHT, while the other has no entry
            elif (0 < valuei < 3 and candidate.loc[i, 'facebook_int'] == 999) or (valuei  == 999 and 0 < candidate.loc[i, 'facebook_int'] < 3):
                print(valuei, candidate.loc[i, 'facebook_int'], "Right")
                candidate.loc[i, 'combine_rating'] = "Right"
             
            # ALL OTHER RATINGS ARE EITHER ABSENT FOR BOTH SCALES OR THE SCALES DISAGREE
            else:
                print(valuei, candidate.loc[i, 'facebook_int'], "Not Rated")
                candidate.loc[i, 'combine_rating'] = "Unknown / unreliable"
                
        except KeyError:
            continue

combine_ratings(HC)

len(HC)

combine_ratings(DT)

"""
Explanation: Next, ratings from Allsides and the Facebook study were combined. Where ratings from both Allsides and Facebook are absent, or disagree, bias rating was decided by Mondo Times where an outlet received more than 20 votes, and /or the author based on outlets' "About" pages and other information. Where doubt remained, or ratings are not applicable (eg. Getty Images), Unknown / Unreliable was assigned.
End of explanation
"""


HC_Unrated = HC[HC.combine_rating == "Unknown / unreliable"]
DT_Unrated = DT[DT.combine_rating == "Unknown / unreliable"]

HC_Unrated.news_source_url.unique()

DT_Unrated.news_source_url.unique()

Unrated_newssource_list = HC_Unrated.news_source_url.unique().tolist()
DT_Unrated_newssource_list = DT_Unrated.news_source_url.unique().tolist()

print(len(Unrated_newssource_list))
print(len(DT_Unrated_newssource_list))

Unrated_newssource_list

DT_Unrated_newssource_list

HC[HC.news_source_url == 'www.youtube.com']

"""
Explanation: Make list of unique news sources, combine lists together from HC and DT, and output to a csv:
End of explanation
"""


Unrated_newssource_list.append('www.youtube.com')

for i in DT_Unrated_newssource_list:
    if i not in Unrated_newssource_list:
        Unrated_newssource_list.append(i)

len(Unrated_newssource_list)

#tmp = pd.DataFrame(Unrated_newssource_list, columns=["news_source"])
#tmp.to_csv('unrated_newssources_imagebox.csv', index=False)

"""
Explanation: Despite YouTube being in the Facebook study, the channel observed here in Hillary Clinton's list is a channel that displays a distinct bias toward Donald Trump and against Hillary Clinton. Therefore, it will be manually rated.
In addition, Reddit, which is also channel-based, will be rated on the specific channel, and not the overall site.
End of explanation
"""


manual_rating = pd.read_csv('unrated_newssources_imagebox.csv')

manual_rating

def merge_manual_ratings(candidate, col):
    
    candidate['final_rating'] = ''
    
    for i, valuei in enumerate(candidate.news_source_url):
        for j, valuej in enumerate(manual_rating.news_source):
            
            if (valuei == valuej):
                print(valuei, valuej, manual_rating.loc[j, col])
                
                try:
                    if manual_rating.loc[j, col] < 0:
                        print("Left")
                        candidate.loc[i, 'final_rating'] = "Left"
                    
                    elif manual_rating.loc[j, col] == 0:
                        print("Center")
                        candidate.loc[i, 'final_rating'] = "Center"
                    
                    elif 999 > manual_rating.loc[j, col] > 0:
                        print("Right")
                        candidate.loc[i, 'final_rating'] = "Right"
                    
                    elif manual_rating.loc[j, col] == 999:
                        print("Unknown/Unreliable")
                        candidate.loc[i, 'final_rating'] = "Unknown / unreliable"
            
                except KeyError:
                    continue
    
    for i, valuei in enumerate(candidate.final_rating):
        if valuei == '':
            try:
                print("currently empty. Let's fill it up!!")
                candidate.loc[i, 'final_rating'] = candidate.loc[i, 'combine_rating']
            except KeyError:
                continue

merge_manual_ratings(HC, 'final_rating_HC')
merge_manual_ratings(DT, 'final_rating_DT')

HC.allsides_bias_rating.value_counts()

DT.allsides_bias_rating.value_counts()

HC.facebookbias_rating.value_counts()

DT.facebookbias_rating.value_counts()

HC.final_rating.value_counts()

DT.final_rating.value_counts()

HC.final_rating.value_counts().plot(kind='bar', alpha=0.5, color='blue')
DT.final_rating.value_counts().plot(kind='bar', alpha=0.5, color='red')

HC.allsides_bias_rating.value_counts().plot(kind='bar', alpha=0.5, color='blue')
DT.allsides_bias_rating.value_counts().plot(kind='bar', alpha=0.5, color='red')

HC[HC.news_source_url == 'russia-insider.com']

DT[DT.final_rating == 'Unknown / unreliable']

HC.to_csv('HC_imagebox_full_ratings.csv', index=False)
DT.to_csv('DT_imagebox_full_ratings.csv', index=False)

HC.columns

"""
Explanation: Read in my manual bias ratings:
End of explanation
"""


