df_ranking=pd.read_csv('article_uni.csv', index_col=0)

print(df_ranking.shape)
df_ranking.head()

df.article_uni.replace('The London School of Economics and Political Science (United-Kingdom)',
    'London School of Economics and Political Science', inplace=True)

from sklearn.preprocessing import MinMaxScaler, StandardScaler

scaler=StandardScaler()

uni_cluster_1=df[df.article_rating==df.article_rating.max()].article_uni.unique()
uni_cluster_2=list(set(df.article_uni.unique())-set(uni_cluster_1))

df[df.article_uni.isin(uni_cluster_1)].article_rating.values*2/10.max()

df['article_rating'][df.article_uni.isin(uni_cluster_1)]=df[df.article_uni.isin(uni_cluster_1)].article_rating.values*2/10#scaler.fit_transform(df[df.article_uni.isin(uni_cluster_1)].article_rating.values)

df['article_rating'][df.article_uni.isin(uni_cluster_2)]=df[df.article_uni.isin(uni_cluster_2)].article_rating.values*2.5/10#scaler.fit_transform(df[df.article_uni.isin(uni_cluster_2)].article_rating.values)

df.article_rating.hist()

"""
Explanation: pd.DataFrame({'article_uni':df.article_uni.unique()}).to_csv('article_uni.csv')
End of explanation
"""


df_ranking.sort_values(['article_uni'],inplace=True)

df_ranking.head()

"""
Explanation: a=pd.pivot_table(df,index=["article_uni"],values=["article_rating"],aggfunc=[len,np.mean], columns='year')
a
End of explanation
"""


b=df[df.article_pub_date>=data_first_ranking].groupby(['article_uni', 'year']).article_rating.agg({'article_rating_mean':'mean',
                                                       'article_rating_count':'count',
                                                        'article_rating_moda':lambda x:x.value_counts().index[0]}).reset_index()
b['ranking']=np.zeros((1, b.shape[0]))[0]
b.head()

for name in b.article_uni.unique():
    for year in b[b.article_uni==name].year:
        #print(year)
        b['ranking'][(b.article_uni==name)&(b.year==year)]=df_ranking[(df_ranking.article_uni==name)][str(year)].values[0]

b=b[(~b.ranking.isnull())]

for year in np.sort(b.year.unique()):
    print(year, round(np.corrcoef(b[(b.year==year)].article_rating_mean, b[(b.year==year)].ranking)[1,0],2))

for year in np.sort(b.year.unique()):
    print(year, round(np.corrcoef(b[(b.year==year)].article_rating_moda, b[(b.year==year)].ranking)[1,0],2))

df_all=pd.merge(b, df_ranking[['article_uni','country']], on=['article_uni'])

df_all.head()

import plotly.plotly as py
from plotly.grid_objs import Grid, Column
from plotly.tools import FigureFactory as FF 

import pandas as pd
import time

years_from_col = set(df_all['year'])
years_ints = sorted(list(years_from_col))
years = [str(year) for year in years_ints]

# make list of continents
countries = []
for country in df_all['country']:
    if country not in countries: 
        countries.append(country)

columns = []
# make grid
for year in years:
    for country in countries:
        df_by_year = df_all[df_all['year'] == int(year)]
        df_by_year_and_cont = df_by_year[df_by_year['country'] == country]
        for col_name in df_by_year_and_cont:
            #print(col_name)
            # each column name is unique
            column_name = '{year}_{country}_{header}_gapminder_grid'.format(
                year=year, country=country, header=col_name
            )
            a_column = Column(list(df_by_year_and_cont[col_name]), column_name)
            columns.append(a_column)

# upload grid
grid = Grid(columns)
url = py.grid_ops.upload(grid, 'gapminder_grid'+str(time.time()), auto_open=False)
url

figure = {
    'data': [],
    'layout': {},
    'frames': [],
    'config': {'scrollzoom': True}
}

# fill in most of layout
figure['layout']['yaxis'] = {'range': [-50, 200], 'title': 'Ranking un THE', 'gridcolor': '#FFFFFF'}
figure['layout']['xaxis'] = {'range': [-0.1, 1.1], 'title': 'Ranking mean', 'gridcolor': '#FFFFFF'}
figure['layout']['hovermode'] = 'closest'
figure['layout']['plot_bgcolor'] = 'rgb(223, 232, 243)'

figure['layout']['sliders'] = {
    'active': 0,
    'yanchor': 'top',
    'xanchor': 'left',
    'currentvalue': {
        'font': {'size': 20},
        'prefix': 'text-before-value-on-display',
        'visible': True,
        'xanchor': 'right'
    },
    'transition': {'duration': 1000, 'easing': 'cubic-in-out'},
    'pad': {'b': 10, 't': 50},
    'len': 0.9,
    'x': 0.1,
    'y': 0,
    'steps': [...]
}

{
    'method': 'animate',
    'label': 'label-for-frame',
    'value': 'value-for-frame(defaults to label)',
    'args': [{'frame': {'duration': 300, 'redraw': False},
         'mode': 'immediate'}
    ],
}

sliders_dict = {
    'active': 0,
    'yanchor': 'top',
    'xanchor': 'left',
    'currentvalue': {
        'font': {'size': 20},
        'prefix': 'Year:',
        'visible': True,
        'xanchor': 'right'
    },
    'transition': {'duration': 300, 'easing': 'cubic-in-out'},
    'pad': {'b': 10, 't': 50},
    'len': 0.9,
    'x': 0.1,
    'y': 0,
    'steps': []
}

figure['layout']['updatemenus'] = [
    {
        'buttons': [
            {
                'args': [None, {'frame': {'duration': 500, 'redraw': False},
                         'fromcurrent': True, 'transition': {'duration': 300, 'easing': 'quadratic-in-out'}}],
                'label': 'Play',
                'method': 'animate'
            },
            {
                'args': [[None], {'frame': {'duration': 0, 'redraw': False}, 'mode': 'immediate',
                'transition': {'duration': 0}}],
                'label': 'Pause',
                'method': 'animate'
            }
        ],
        'direction': 'left',
        'pad': {'r': 10, 't': 87},
        'showactive': False,
        'type': 'buttons',
        'x': 0.1,
        'xanchor': 'right',
        'y': 0,
        'yanchor': 'top'
    }
]

custom_colors = {
    'UK': 'rgb(171, 99, 250)',
    'USA': 'rgb(230, 99, 250)',
    'Canada': 'rgb(99, 110, 250)',
}

"""
Explanation: b=df[df.article_pub_date>=data_first_ranking].groupby(['article_uni', 'year']).article_rating.agg(lambda x:x.value_counts().index[0]).reset_index()
End of explanation
"""


df_all.country.unique()

col_name_template = '{year}_{country}_{header}_gapminder_grid'
year = df_all.year.min()
for country in countries:
    data_dict = {
        'xsrc': grid.get_column_reference(col_name_template.format(
            year=year, country=country, header='article_rating_mean'
        )),
        'ysrc': grid.get_column_reference(col_name_template.format(
            year=year, country=country, header='ranking'
        )),
        'mode': 'markers',
        'textsrc': grid.get_column_reference(col_name_template.format(
            year=year, country=country, header='article_uni'
        )),
        'marker': {
            'sizemode': 'area',
            'sizeref': 0.05,
            'sizesrc': grid.get_column_reference(col_name_template.format(
                 year=year, country=country, header='article_rating_count'
            )),
            'color': custom_colors[country]
        },
        'name': country
    }
    figure['data'].append(data_dict)

for year in years:
    frame = {'data': [], 'name': str(year)}
    for country in countries:
        data_dict = {
            'xsrc': grid.get_column_reference(col_name_template.format(
                year=year, country=country, header='article_rating_mean'
            )),
            'ysrc': grid.get_column_reference(col_name_template.format(
                year=year, country=country, header='ranking'
            )),
            'mode': 'markers',
            'textsrc': grid.get_column_reference(col_name_template.format(
                year=year, country=country, header='article_uni', 
                )),
            'marker': {
                'sizemode': 'area',
                'sizeref': 0.05,
                'sizesrc': grid.get_column_reference(col_name_template.format(
                    year=year, country=country, header='article_rating_count'
                )),
                'color': custom_colors[country]
            },
            'name': country
        }
        frame['data'].append(data_dict)

    figure['frames'].append(frame)
    slider_step = {'args': [
        [year],
        {'frame': {'duration': 300, 'redraw': False},
         'mode': 'immediate',
       'transition': {'duration': 300}}
     ],
     'label': year,
     'method': 'animate'}
    sliders_dict['steps'].append(slider_step)

figure['layout']['sliders'] = [sliders_dict]

py.icreate_animations(figure, 'gapminder_example'+str(time.time()))

import seaborn as sns

df_all.head()

for i in  np.sort(df_all.year.unique()):
    plt.scatter(df_all[df_all.year==i].article_rating_mean,
           df_all[df_all.year==i].ranking)
    plt.title('Corr between ranking in THE and ranking review in {:0.0f}'.format(i))
    plt.show()

df_all.article_rating_count

for i in  np.sort(df_all.year.unique()):
    plt.scatter(df_all[df_all.year==i].article_rating_count,
           df_all[df_all.year==i].ranking)
    plt.title('Corr between ranking in THE and ranking review count in {:0.0f}'.format(i))
    plt.show()

plt.hist(df_all.article_rating_count)

import selenium

from selenium import webdriver

browser = webdriver.Chrome(executable_path="..\\chromedriver.exe")

"""
Explanation: column_name = '{year}{country}{header}_gapminder_grid'.format(
                year=year, country=country, header=col_name
            )
End of explanation
"""


url = "https://www.niche.com/colleges/stanford-university/reviews/"
browser.get(url)

browser.find_element_by_css_selector('.icon-arrowright-thin--pagination').click()

browser.page_source

element.get_attribute('innerHTML')

"""
Explanation: browser = webdriver.Edge(executable_path="..\MicrosoftWebDriver.exe")
End of explanation
"""


