from IPython.display import Image

Image('./dataframe.jpg')

"""
Explanation: Introduction to Data Analysis and Plotting in Python
This tutorial is...
* introductory and geared towards people new to Python, but familiar with STATA, MATLAB, R, etc.
* interactive with simple exercises along the way
We'll be introducing Pandas for data handling and analysis, and using Matplotlib, Seaborn and Bokeh for visualization
 Please execute sequentially 
What is pandas?
Pandas provides objects for working with data - Series and DataFrame
Data structures have methods for manipulating data eg. indexing, sorting, grouping, filling in missing data
Pandas does not provide modeling tools eg. regression, prediction
* These tools are found in packages such as scikit-learn and statsmodels, which are built on top of pandas
DataFrames
A DataFrame combines multiple 'columns' of data into a two-dimensional object, similar to a spreadsheet
End of explanation
"""


import pandas as pd  # Import the package

data = pd.read_csv('data.csv')
data.head()

data.info()

"""
Explanation: We will create a DataFrame by reading in a CSV file and assigning it to the variable name data
Info on the data set

The data come from 'Assessing the Rate of Replication in Economics', American Economics Review: Papers & Proceedings, 2017
The dataframe data contains 70 empirical papers from AER's 100th volume
Web of Science: number of Web of Science citations
Top 200: number of Top 200 economics journal citations
Google Scholar: number of Google Scholar citations
Replications: number of citations that are replications of the paper
Extensions: number of citations that are extensions of the paper
Both: number of citations that are either replications or extensions of the paper
End of explanation
"""


data['Top 200'].head()

"""
Explanation: We can access individual columns of data, returning a Series
End of explanation
"""


data[['Title', 'Top 200']].head()

"""
Explanation: We can also select multiple columns, returning a new dataframe
End of explanation
"""


data['Total'] = data['Web of Science'] + data['Top 200'] + data['Google Scholar']
data.head()

"""
Explanation: We can add a new column to our dataframe like so
End of explanation
"""


data.mean()

"""
Explanation: If we want to know the average number of Web of Science citations...
End of explanation
"""


data.describe()

"""
Explanation: .describe() returns useful summary statistics
End of explanation
"""


data[:5]

"""
Explanation: Exercises
Find the means of Top 200 and Google Scholar citations
<button data-toggle="collapse" data-target="#sol2" class='btn btn-primary'>Solution</button>
<div id="sol2" class="collapse">
```python
data[['Top 200', 'Google Scholar']].mean()
```
```
Top 200            22.257143
Google Scholar    227.571429
dtype: float64
```

#### Create a table of summary statistics of Top 200 citations and Google Scholar citations

<button data-toggle="collapse" data-target="#sol4" class='btn btn-primary'>Solution</button>
<div id="sol4" class="collapse">
```python
data[['Top 200', 'Google Scholar']].describe()
```
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Top 200</th>
      <th>Google Scholar</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>count</th>
      <td>70.000000</td>
      <td>70.000000</td>
    </tr>
    <tr>
      <th>mean</th>
      <td>22.257143</td>
      <td>227.571429</td>
    </tr>
    <tr>
      <th>std</th>
      <td>23.182131</td>
      <td>257.258375</td>
    </tr>
    <tr>
      <th>min</th>
      <td>0.000000</td>
      <td>7.000000</td>
    </tr>
    <tr>
      <th>25%</th>
      <td>9.000000</td>
      <td>73.250000</td>
    </tr>
    <tr>
      <th>50%</th>
      <td>15.000000</td>
      <td>139.000000</td>
    </tr>
    <tr>
      <th>75%</th>
      <td>23.750000</td>
      <td>267.250000</td>
    </tr>
    <tr>
      <th>max</th>
      <td>108.000000</td>
      <td>1246.000000</td>
    </tr>
  </tbody>
</table>

#### Create a table of summary statistics of the paper's fields (according to the variable `Field`)

<button data-toggle="collapse" data-target="#sol5" class='btn btn-primary'>Solution</button>
<div id="sol5" class="collapse">
```python
data['Field'].describe()
```
```
count                  70
unique                  5
top       Macro/Int/Trade
freq                   25
Name: Field, dtype: object
```

## Selecting and filtering

We can use integer slicing to select rows as follows
End of explanation
"""


pd.set_option('max_colwidth', 100)  # adjust column width

data[['Title', 'Google Scholar']].sort_values(by='Google Scholar', ascending=False)[:5]

"""
Explanation: We might want to find the top cited papers
First we will sort our values by a column in the dataframe
End of explanation
"""


data.set_index('Title').head()

"""
Explanation: Another way to select rows is to use row labels, ie. set a row index
Similar to the column labels, we can add row labels (the index)
End of explanation
"""


data.head()

"""
Explanation: Note: we haven't actually changed data
End of explanation
"""


# data.reset_index(inplace=True)   # Uncomment this if you receive an error

data = data.set_index('Title')   # Can also use data.set_index('Title', inplace=True)
data.head()

data.loc['Matching and Sorting in Online Dating']

data.loc[['Matching and Sorting in Online Dating', 
          'Learning about a New Technology: Pineapple in Ghana']]

"""
Explanation: We need to reassign the variable name data to the new copy
End of explanation
"""


data.reset_index(inplace=True)

data['Field'] == 'Development'

"""
Explanation: Alternatively, we can filter our dataframe (select rows) using boolean conditions
End of explanation
"""


data[data['Field'] == 'Development']

data[(data['Field'] == 'Development') & (data['Web of Science'] > 100)]

"""
Explanation: Selecting rows with this boolean condition will return only rows of the dataframe where Field == 'Development' is True
End of explanation
"""


data.reset_index(inplace=True)

grouped = data.groupby('Field')
grouped

"""
Explanation: Exercises
How many papers in the sample have over 100 Google Scholar citations?

Hint: use len() to find the length of a dataframe

<button data-toggle="collapse" data-target="#sol6" class='btn btn-primary'>Solution</button>
<div id="sol6" class="collapse">
```python
len(data[data['Google Scholar'] > 100])
```
```
44
```

#### How many 'Labor/IO' papers have over 100 Google Scholar citations?

<button data-toggle="collapse" data-target="#sol7" class='btn btn-primary'>Solution</button>
<div id="sol7" class="collapse">
```python
len(data[(data['Google Scholar'] > 100) & (data['Field'] == 'Labor/IO')])
```
```
10
```

## Grouping and aggregating data

We might want to summarize our data by grouping it by fields

To do this, we will use the `.groupby()` function
End of explanation
"""


grouped.mean()

grouped['Both'].mean()

grouped['Both'].agg(['mean', 'median', 'count'])

"""
Explanation: To return an aggregated dataframe, we need to specify the function we would like pandas to use to aggregate our groups
End of explanation
"""


import matplotlib.pyplot as plt

grouped['Google Scholar'].mean().plot(kind='bar')
plt.show()

data.plot(x='Google Scholar', y='Both', kind='scatter', alpha=0.6)
plt.ylabel('Total replications and extensions')
plt.xlabel('Number of citations')
plt.grid()
plt.show()

"""
Explanation: A list of built-in aggregatation functions can be found here
Plotting
Pandas provides a built-in plotting command to create matplotlib plots from your dataframes
To display our plots, we need to import first matplotlib
End of explanation
"""


import seaborn as sns  # Import the package

sns.jointplot(x='Google Scholar', y='Both', data=data)
plt.show()

sns.kdeplot(data=data[data['Replications'] > 0].loc[:, 'Web of Science'], cumulative=True, bw=2)
sns.kdeplot(data=data[data['Replications'] == 0].loc[:, 'Web of Science'], cumulative=True, linestyle='--', bw=2)
plt.xlim(0, 200)
plt.xlabel('Web of Science Citations')
plt.ylabel('CDF')
plt.legend(['One or more replications', 'No replications'])
plt.show()

"""
Explanation: Exercise
Plot the average number of replications and extensions for each field, sorted from highest to lowest
<button data-toggle="collapse" data-target="#sol9" class='btn btn-primary'>Solution</button>
<div id="sol9" class="collapse">
```python
grouped['Both'].mean().sort_values(ascending=False).plot(kind='bar')
plt.title('Average number of replications/extensions by field')
plt.show()
```
<img src = './plot_groupby.png'>

## Another plotting library: `seaborn`

* `seaborn` is a plotting library built on top of `matplotlib`

* It is geared towards producing pretty plots for statistical applications

* You can find an example gallery of `seaborn` plots [here](https://seaborn.pydata.org/examples/index.html)
End of explanation
"""


from bokeh.plotting import figure, output_notebook, show
from bokeh.models import ColumnDataSource, HoverTool
output_notebook()

# Add data source
source = ColumnDataSource(data[['Title', 'Web of Science', 'Replications', 'Extensions']])
colors = ['red', 'blue']

# Add title tooltips
hover = HoverTool(tooltips=[
    ("Title", "@Title"),
    ("(x,y)", "($x, $y)")
])

# Create figure
p = figure(tools=[hover])

# Plot scatter
for to_plot, color in zip(['Replications', 'Extensions'], colors):
    p.circle(x='Web of Science', y=to_plot, source=source, size=7, alpha=0.5, legend=[to_plot.title()], color=color)

p.legend.click_policy = 'hide'
p.xaxis.axis_label = 'Number of citations'
p.yaxis.axis_label = 'Number of replications or extensions'

show(p)

"""
Explanation: Bokeh example
Bokeh is a Python library that makes creating interactive plots super easy - an example gallery is here
End of explanation
"""


