# examines distribution of number of words

df_online['reviews'].fillna(0).plot.hist(normed=True, 
                                         bins=np.arange(0, 50, 1), alpha=0.5, histtype='step', linewidth='2')
df_online['favs'].fillna(0).plot.hist(normed=True, 
                                         bins=np.arange(0, 50, 1), alpha=0.5, histtype='step', linewidth='2')
df_online['follows'].fillna(0).plot.hist(normed=True, 
                                         bins=np.arange(0, 50, 1), alpha=0.5, histtype='step', linewidth='2')
plt.xlim(0,50)
plt.legend().set_visible(True)

plt.show()

"""
Explanation: Fanfiction Story Analysis
Performance benchmarking and prediction
The success of a story is typically judged by the number of reviews, favorites, or followers it recieves. Here, we will try to predict how successful a story will be given select observable features, as well as develop a way to benchmark existing stories. That is, if we were given a story's features, we can determine whether that story is overperforming or underperforming relative to its peers.
Reviews, favorites, and follows
First and foremost, let us examine the distribution of each of these "success" metrics.
End of explanation
"""


df_online.columns.values

# creates regressand variables
df_online['ratedM'] = [row == 'M' for row in df_online['rated']]
df_online['age'] = [cyear - int(row) for row in df_online['pub_year']]
df_online['fansize'] = [fandom[row] for row in df_online['fandom']]
df_online['complete'] = [row == 'Complete' for row in df_online['status']]
df_online['lnchapters'] = np.log(df_online['chapters'])

# creates independent variables
df_online['lnreviews'] = np.log(df_online['reviews']+1)
df_online['lnfavs'] = np.log(df_online['favs']+1)
df_online['lnfollows'] = np.log(df_online['follows']+1)

df_online['lnfavs'] = np.log(df_online['favs']+1)

sns.pairplot(data=df_online, y_vars=['lnfavs'], x_vars=['lnchapters', 'lnwords1k', 'age'])
sns.pairplot(data=df_online, y_vars=['favs'], x_vars=['chapters', 'words', 'age'])

plt.show()



sns.pairplot(data=df_online, y_vars=['lnreviews'], x_vars=['lnchapters', 'lnwords1k', 'age'])
sns.pairplot(data=df_online, y_vars=['reviews'], x_vars=['chapters', 'words', 'age'])

plt.show()

# runs OLS regression
formula = 'reviews ~ chapters + words1k + ratedM + age + fansize + complete'
reg = smf.ols(data=df_online, formula=formula).fit()
print(reg.summary())

# runs OLS regression
formula = 'lnreviews ~ lnchapters + lnwords1k + ratedM + age + fansize + complete'
reg = smf.ols(data=df_online, formula=formula).fit()
print(reg.summary())

# runs OLS regression
formula = 'lnfavs ~ lnchapters + lnwords1k + ratedM + age + fansize'
reg = smf.ols(data=df_online, formula=formula).fit()
print(reg.summary())

# creates copy of only active users
df_active = df_profile.loc[df_profile.status != 'inactive', ].copy()

# creates age variable
df_active['age'] = 17 - pd.to_numeric(df_active['join_year'])
df_active.loc[df_active.age < 0, 'age'] = df_active.loc[df_active.age < 0, 'age'] + 100
df_active = df_active[['st', 'fa', 'fs', 'cc', 'age']]

# turns cc into binary
df_active.loc[df_active['cc'] > 0, 'cc'] = 1

"""
Explanation: As expected, reviews, favorites, and follows all have heavily right-skewing distributions. However, there are also differences. A story is mostly likely to have 1 or 2 reviews, not 0. A story is mostly likely to have 0 favorites, but otherwise the favorites distribution looks very similar to reviews. Follows is the one that deviates the most. About one-fourth of stories have 0 or 1 follows. 
We assumed authors prefer having reviews first and foremost, then favorites, then follows. The data reveals that it is actually follows that is the most "rare" out of the three metrics, then favorites, and finally review. 
This is accordance with intuition. Anyone can sign a review, with or without an account. Only users with accounts can increase a story's favorite counter. Finally, follows are the most "hassling", as they send update messages to a follower's email inbox. Consequently, they are the least common.
End of explanation
"""


# displays correlation matrix
df_active.corr()

# creates design_matrix 
X = df_active
X['intercept'] = 1

# displays variance inflation factor
vif_results = pd.DataFrame()
vif_results['VIF Factor'] = [vif(X.values, i) for i in range(X.shape[1])]
vif_results['features'] = X.columns
vif_results

"""
Explanation: Multicollinearity
End of explanation
"""


# runs OLS regression
formula = 'st ~ fa + fs + cc + age'
reg = smf.ols(data=df_active, formula=formula).fit()
print(reg.summary())

"""
Explanation: Results indicate there is some correlation between two of the independent variables: 'fa' and 'fs', implying one of them may not be necessary in the model.
Nonlinearity
We know from earlier distributions that some of the variables are heavily right-skewed. We created some scatter plots to confirm that the assumption of linearity holds.
The data is clustered around the zeros. Let's try a log transformation.
Regression Model
End of explanation
"""


# runs OLS regression
formula = 'st ~ fa + cc + age'
reg = smf.ols(data=df_active, formula=formula).fit()
print(reg.summary())

"""
Explanation: The log transformations helped increase the fit from and R-squared of ~0.05 to ~0.20.
From these results, we can see that:

A 1% change in number of authors favorited is associated with a ~15% change in the number of stories written.
A 1% change in number of stories favorited is associated with a ~4% change in the number of stories written.
Being in a community is associated with a ~0.7 increase in the number of stories written.
One more year on the site is associated with a ~3% change in the number of stories written.

We noted earlier that 'fa' and 'fs' had a correlation of ~0.7. As such, we reran the regression without 'fa' first, then again without 'fs'. The model without 'fs' yielded a better fit (R-squared), as well as AIC and BIC.
End of explanation
"""


def graph(formula, x_range):  
    y = np.array(x_range)
    x = formula(y)
    plt.plot(y,x)  

graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1)))), 
      range(2,100,1))
graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1))+reg.params[2])), 
      range(2,100,1))

plt.show() 

ages = [0, 1, 5, 10, 15]
for age in ages:
    graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1))+reg.params[3]*age)), 
          range(2,100,1))

plt.show() 

"""
Explanation: Without 'fs', we lost some information but not much:

A 1% change in number of authors favorited is associated with a ~20% change in the number of stories written.
Being in a community is associated with a ~0.7 increase in the number of stories written.
One more year on the site is associated with a ~3% change in the number of stories written.

All these results seem to confirm a basic intuition that the more active an user reads (as measured by favoriting authors and stories), the likely it is that user will write more stories. Being longer on the site and being part of a community is also correlated to publications.
To get a sense of the actual magnitude of these effects, let's attempt some plots:
End of explanation
"""


