import numpy as np
import scipy.stats as stats
import csv
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline


satscores = '/Users/DES/DSI-NYC-5/Projects/project-1-sat-scores/assets/sat_scores.csv'


rows = []
with open(satscores, 'r') as f:
    reader = csv.reader(f)
    for row in reader:
        rows.append(row)


"""
Explanation: Project 1
Step 1: Open the sat_scores.csv file. Investigate the data, and answer the questions below.
1. What does the data describe?
The sat-score data describes SAT, standardized college entrance exam, scores from the 50 states and the District of Columbia for test taking year 2001, as provided by the College Board. This data also contains participation in the SAT exam, presumably in percentages  and across the U.S. Finally, the last row of data is an aggregator of all 50 states, plus DC, for participation rates and SAT score where verbal and math are separately stated.
2. Does the data look complete? Are there any obvious issues with the observations?
The data does contain a complete listing of SAT scores for all states plus the District of Columbia. The last row contains the nationwide SAT scores and participation, which is not to be included in the 50 state, plus DC view of the data. Additionally, another issue with the data is the unpacked version of the SAT scores given. Therefore, both verbal and math scores are summed in order to get total SAT score.
3. Create a data dictionary for the dataset.
SAT Scores in 2001
Description
The sat-score data describes SAT, standardized college entrance exam, scores from the 50 states and the District of Columbia for test taking year 2001, as provided by the College Board. This data also contains participation in the SAT exam, presumably in percentages. Finally, the last row of data is an aggregator of all 50 states, plus DC, for participation rates and SAT score where verbal and math are separately stated.
Methodology
Format a panda dataframe from a comma delimited file containing 51 observations on the following 4 variables.
State 
50 states of the U.S, plus the District of Columbia
Rate
Test participation rate; denoted in percentage by State
Verbal
Result of Verbal component of the SAT exam; section graded on a scale of 200–800
Math
Result of Math component of the SAT exam; section graded on a scale of 200–800
Total SAT 
Calculated from source data.  Combines the Math and Verbal components of the exam issued in 2001.
Step 2: Load the data.
4. Load the data into a list of lists
End of explanation
"""


print rows


"""
Explanation: 5. Print the data
End of explanation
"""


#Header is list of labels from data
header = rows[0]
header


#Data minus Header list
data = rows[1:]
data[0:10]


"""
Explanation: 6. Extract a list of the labels from the data, and remove them from the data.
End of explanation
"""


#Exclusive List of States
list_states =[]

for t in data:
    list_states.append(t[0])


list_states[0:10]


#List of Lists of Rate, SAT Scores
scores_rate = []

for t in data:
    scores_rate.append(t[1:])
    

scores_rate[0:10]


"""
Explanation: 7. Create a list of State names extracted from the data. (Hint: use the list of labels to index on the State column)
End of explanation
"""


type(scores_rate)


"""
Explanation: 8. Print the types of each column
End of explanation
"""


scores_rate[0:10]


numerical_list = []
index = []

for x in scores_rate:
    index = list(map(int, x))
    numerical_list.append(index)


print numerical_list[0:10]


type(numerical_list)


"""
Explanation: 9. Do any types need to be reassigned? If so, go ahead and do it.
End of explanation
"""


header

header_m_s = header[1:]
header_m_s


numerical_list[0:10]

sat_data = {}

for name in header_m_s:
    sat_data[name] = [x[header_m_s.index(name)] for x in numerical_list]


sat_data.values()


type(sat_data)


sat_data.keys()


type(list_states)


"""
Explanation: 10. Create a dictionary for each column mapping the State to its respective value for that column.
End of explanation
"""


sat_data['Math'][0:10]


for i, j in sat_data.items():
    j = [float(x) for x in j]
    sat_data[i] = j


sat_data['Math'][0:10]


sat_data.keys()

temp = []
dictlist = []

#convert dictionary to list
for key, value in sat_data.iteritems():
    temp = [key,value]
    dictlist.append(temp)


dictlist

"""
Explanation: 11. Create a dictionary with the values for each of the numeric columns
End of explanation
"""


import pandas as pd
satscores = pd.read_csv('/Users/DES/DSI-NYC-5/Projects/project-1-sat-scores/assets/sat_scores.csv')


satscores.head()


sat = pd.DataFrame(sat, columns=['State','Rate','Verbal','Math','Total_SAT'])


#Exclude the 'ALL' category from data
sats = sat.iloc[:51]

sat['Total_SAT'] = sat['Verbal'] + sat['Math']       #Included an aggregate version of SAT
sat[0:10]


print "Participation Rate Min:",sats["Rate"].min()


print "Participation Rate Max:",sats["Rate"].max()


print "SAT Math Min:",sats["Math"].min()


print "SAT Math Max:",sats["Math"].max()


print "SAT Verbal Min:",sat["Verbal"].min()


print "SAT Verbal Max:",sats["Verbal"].max()


print "Total SAT Min:",sat["Total_SAT"].min()


print "Total SAT Max:",sats["Total_SAT"].max()


def summary_stats(col, data):
    print 'COLUMN: ' + col
    print 'mean: ' + str(np.mean(data))
    print 'median: ' + str(np.median(data))
    print 'mode: ' + str(stats.mode([round(d) for d in data]))
    print 'variance: ' + str(np.var(data))
    print 'standard deviation: ' + str(np.std(data))


summary_stats('Rate', sats['Rate'])


summary_stats('Math', sats['Math'])


summary_stats('Verbal', sats['Verbal'])


summary_stats('Total_SAT', sats['Total_SAT'])


"""
Explanation: Step 3: Describe the data
12. Print the min and max of each column
End of explanation
"""


def stddev(data):
    """returns the standard deviation of lst"""
    m = np.mean(data)
    variance = sum([(i - m)**2 for i in data]) / len(data)
    return np.sqrt(variance)


stddev(sats['Rate'])


stddev(sats['Math'])


stddev(sats['Verbal'])


stddev(sats['Total_SAT'])


#Hypothesis testing where
# H0 (null hypothesis): There is no difference between Math and Verbal SAT Scores
# HA (alternative hypothesis): There is a difference between Math and Verbal SAT Scores

a_mean = sats['Math'].mean()
b_mean = sats['Verbal'].mean()

a_var = sats['Math'].var()
b_var = sats['Verbal'].var()

a_n = len(sats['Math'])
b_n = len(sats['Verbal'])

numerator = a_mean - b_mean
denominator = np.sqrt((a_var / a_n) + (b_var / b_n))
z = numerator / denominator
z


p_val = 1 - stats.norm.cdf(z)
p_val


alpha = .01
print p_val, alpha, p_val > alpha


"""
Explanation: 13. Write a function using only list comprehensions, no loops, to compute Standard Deviation. Print the Standard Deviation of each numeric column.
End of explanation
"""


ax = sns.distplot(sats['Rate'], bins=10)
sns.distplot(sats['Rate'], color='darkred', bins=10, ax=ax)

ax = plt.axes()
ax.set_title('Distribution SAT Participation Rate')
plt.show()


"""
Explanation: Step 4: Visualize the data
14. Using MatPlotLib and PyPlot, plot the distribution of the Rate using histograms.
End of explanation
"""


ax = sns.distplot(sats['Math'], bins=10)
sns.distplot(sats['Math'], color='yellow', bins=10, ax=ax)

ax = plt.axes()
ax.set_title('Distribution of Math SAT Scores')
plt.show()


"""
Explanation: 15. Plot the Math distribution
End of explanation
"""


ax = sns.distplot(sats['Verbal'], bins=10)
sns.distplot(sats['Verbal'], color='darkblue', bins=10, ax=ax)

ax = plt.axes()
ax.set_title('Distribution of Verbal SAT Scores')
plt.show()


ax = sns.distplot(sats['Total_SAT'], bins=10)
sns.distplot(sats['Total_SAT'], color='darkblue', bins=10, ax=ax)

ax = plt.axes()
ax.set_title('Distribution of Total SAT Scores')
plt.show()


"""
Explanation: 16. Plot the Verbal distribution
End of explanation
"""


import seaborn as sns
sns.pairplot(sats)
plt.show()


"""
Explanation: 17. What is the typical assumption for data distribution?
A typical assumption of a data distribution is that the distribution is normal or the data is bell-curve shaped.
18. Does that distribution hold true for our data?
No, these numeric fields do not have a normal distribution. The SAT Verbal component is negatively skewed, whereas both Participation Rate and SAT Math are right skewed or positively skewed distribution.
19. Plot some scatterplots. BONUS: Use a PyPlot figure to present multiple plots at once.
End of explanation
"""


data = [sats['Math'], sats['Verbal']]

fig, ax1 = plt.subplots(figsize=(12, 8))

plt.boxplot(data)

ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
               alpha=0.5)

ax1.set_axisbelow(True)
ax1.set_title('Box Plot of SAT Math / Verbal Scores', y =1.03, fontsize = 24)
ax1.set_xlabel('Features', fontsize = 18)
ax1.set_ylabel('SAT Scores', fontsize = 18)

# Set the axes ranges and axes labels
numBoxes = 2
ax1.set_xlim(0.5, numBoxes + 0.5) 
ax1.set_ylim(400, 625)
xtickNames = plt.setp(ax1, xticklabels=['SAT Math Score', 'SAT Verbal Score'])
plt.setp(xtickNames, fontsize=14)

plt.axhline(625, color = 'darkgreen')
plt.axvline(1, color = 'darkgreen', linewidth = 1, alpha = 0.4)

plt.show()


data = [sats['Total_SAT']]

fig, ax1 = plt.subplots(figsize=(12, 8))

plt.boxplot(data)

ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
               alpha=0.5)

ax1.set_axisbelow(True)
ax1.set_title('Box Plot of Total SAT Scores', y =1.03, fontsize = 24)
ax1.set_xlabel('Feature', fontsize = 18)
ax1.set_ylabel('Combined SAT Scores', fontsize = 18)

# Set the axes ranges and axes labels
numBoxes = 1
ax1.set_xlim(0.5, numBoxes + 0.5) 
ax1.set_ylim(900, 1300)
xtickNames = plt.setp(ax1, xticklabels=['Total SAT Scores'])
plt.setp(xtickNames, fontsize=14)

plt.axhline(1300, color = 'darkgreen')
plt.axvline(1, color = 'darkgreen', linewidth = 1, alpha = 0.4)

plt.show()

data = [sats['Rate']]

fig, ax1 = plt.subplots(figsize=(12, 8))

plt.boxplot(data)

ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
               alpha=0.5)

ax1.set_axisbelow(True)
ax1.set_title('Box Plot of Participation Rate in SAT Examination', y =1.03, fontsize = 24)
ax1.set_xlabel('Feature', fontsize = 18)
ax1.set_ylabel('Participation Rate', fontsize = 18)

# Set the axes ranges and axes labels
numBoxes = 1
ax1.set_xlim(0.5, numBoxes + 0.5) 
ax1.set_ylim(0, 100)
xtickNames = plt.setp(ax1, xticklabels=['Participation Rate'])
plt.setp(xtickNames, fontsize=14)

plt.axhline(100, color = 'darkgreen')
plt.axvline(1, color = 'darkgreen', linewidth = 1, alpha = 0.4)

plt.show()


"""
Explanation: 20. Are there any interesting relationships to note?
There seems to be a suggestive proportional relationship between SAT Math, SAT Verbal and Total scores, overall.   That is, for example, as verbal scores increase, the math scores proportionally and positively increase. Other variable relationships, however, seem to have a rather inconclusive linear relationship.  When considering building a linear regression model to describe Math, Verbal or Total Score one would need to address the outliers the scatter plots above display for each resective scores.
21. Create box plots for each variable.
End of explanation
"""


sat.to_csv("/Users/DES/DSI-NYC-5/Projects/project-1-sat-scores/assets/SAT_Scores_DC.csv", sep='\t')


"""
Explanation: BONUS: Using Tableau, create a heat map for each variable using a map of the US.
End of explanation
"""


