print('Hello World!') # Python 3
print 'Hello World!'  # Python 2

# An object with no functions or operators is also printed to the console
x = 'Hello World!'
x

"""
Explanation: License

Copyright (C) 2017 J. Patrick Hall, jphall@gwu.edu
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Python: Part 0 - Pandas and Numpy
1. Standard output
print is the primary function used to write to the console in Python
* print is a function in Python 3
* print is a statement in Python 2
End of explanation
"""


# import packages
import string                   # module with string utilities
import pandas as pd             # large module with many utilities for dataframes, here aliased as 'pd' 
import numpy as np              # large module with many numeric and mathematical utilities, here aliased as 'np'
import matplotlib.pyplot as plt # module for plotting

# "magic" syntax to display matplotlib graphics in a notebook
# magic statements start with '%' and are often used to control notebook behavior
%matplotlib inline             

"""
Explanation: 2. Importing libraries
Python contains many libraries, often called modules, for different purposes
Modules are:
* Nearly always free and open source
* Installed using many different methods - a package manager like conda, readily available through the Anaconda release of Python (https://www.continuum.io/downloads) - is often a good solution for installing and managing packages/modules 
* Of relatively high and uniform quality and but licensing can vary
* Imported using the import statement
End of explanation
"""


n_rows = 1000
n_vars = 2

"""
Explanation: 3. Generating a sample data set
Set the number of rows and columns for the sample data set
End of explanation
"""


# list comprehension
# str() converts to string
# range() creates a list of values from arg1 to arg2
num_col_names = ['numeric' + str(i+1) for i in range(0, n_vars)]  
num_col_names

type(num_col_names) # type() can be used to determine the class of an object in Python

"""
Explanation: Create lists of strings that will become column names

Lists are a common data structure in Python
Lists are surrounded by square brackets [] and contain different data types as list elements
Lists can be created by a speficic type Pythonic syntax, called list comprehensions
Lists in Python are indexed from 0, unlike SAS or R
Lists in Python, and other data structures, can be sliced using numeric indices
End of explanation
"""


# anonymous functions
# the lamba statement is used to define simple anonymous functions
# map() is very similar to to lapply() in R - it applies a function to the elements of a list
char_col_names = map(lambda j: 'char' + str(j+1), range(0, n_vars)) 
char_col_names

"""
Explanation: Python supports anonymous functions
End of explanation
"""


# string.ascii_uppercase is a string constant of uppercase letters
print(string.ascii_uppercase)

# another list comprehension
# slice first seven letters of the string
text_draw = [(letter * 8) for letter in string.ascii_uppercase[:7]] 
text_draw

"""
Explanation: Create a list of text elements from which to sample
End of explanation
"""


randoms = np.random.randn(n_rows, n_vars)
randoms[0:5]

type(randoms)

"""
Explanation: Create a random numerical columns directly using numpy
The numerical columns will originally be a 2-D numpy array
End of explanation
"""


num_cols = pd.DataFrame(randoms, columns=num_col_names)
num_cols.head()

type(num_cols)

"""
Explanation: Create numerical columns of Pandas dataframe from numpy array
Notice that a key is generated automatically
End of explanation
"""


char_cols = pd.DataFrame(np.random.choice(text_draw, (n_rows, n_vars)), 
                         columns=char_col_names)
char_cols.head()

"""
Explanation: Create random character columns as a Pandas dataframe

Use a numpy sampling function choice() to generate a 2-D numpy array of random text values
Create Pandas dataframe from numpy 2-D array
End of explanation
"""


scratch_df = pd.concat([num_cols, char_cols], axis=1)
scratch_df.head()

"""
Explanation: Use Pandas concat() function to join the numeric and character columns into a new dataframe
End of explanation
"""


# Pandas alllows slicing by dataframes index using ix[]
# ix[:, 0] means all rows of the 0th column - or numeric1
scratch_df.ix[:, 0].plot.hist(title='Histogram of Numeric1')

"""
Explanation: 4. Plotting variables in a dataframe
Pandas has several builtin plotting utilities
Use Pandas hist() method to plot a histogram of numeric1
End of explanation
"""


scratch_df.plot.scatter(x='numeric1', y='numeric2',
                        title='Numeric1 vs. Numeric2')

"""
Explanation: use Pandas scatter() method to plot numeric1 vs. numeric2
End of explanation
"""


# one column returns a Pandas series
# a Pandas series is like a single column vector
scratch_df.iloc[:, 0].head()

type(scratch_df.iloc[:, 0])

# more than one columns makes a dataframe
# iloc enables location by index
scratch_df.iloc[:, 0:2].head()

type(scratch_df.iloc[:, 0:2])

"""
Explanation: 5. Subsetting Pandas dataframes
By columns
Subsetting by index
End of explanation
"""


scratch_df['numeric1'].head()

scratch_df.numeric1.head()

# loc[] allows for location by column or row label 
scratch_df.loc[:, 'numeric1'].head()

# loc can accept lists as an input
scratch_df.loc[:, ['numeric1', 'numeric2']].head()

"""
Explanation: Subsetting by variable name
End of explanation
"""


scratch_df[0:3]

# Selecting by index 
scratch_df.iloc[0:5, :] 

# select by row label
# here index/key values 0:5 are returned
scratch_df.loc[0:5, :]

"""
Explanation: By rows
Subsetting by index
End of explanation
"""


scratch_df[scratch_df.numeric2 > 0].head()

scratch_df[scratch_df.char1 == 'AAAAAAAA'].head()

scratch_df[scratch_df.char1.isin(['AAAAAAAA', 'BBBBBBBB'])].head()

scratch_df[scratch_df.numeric2 > 0].loc[5:10, 'char2']

"""
Explanation: Boolean subsetting
End of explanation
"""


# must use .copy() or this will be a symbolic link
scratch_df2 = scratch_df.copy()

# Pandas supports in place overwrites of data
# overwrite last 500 rows of char1 with ZZZZZZZZ
scratch_df2.loc[500:, 'char1'] = 'ZZZZZZZZ'
scratch_df2.tail()

# iat[] allows for fast location of specific indices
scratch_df2.iat[0, 0] = 1000
scratch_df2.head()

"""
Explanation: 6. Updating the dataframe
End of explanation
"""


scratch_df2.sort_values(by='char1').head()

"""
Explanation: 7. Sorting the dataframe
Sort by values of one variable
End of explanation
"""


scratch_df3 = scratch_df2.sort_values(by=['char1', 'numeric1'],
                            ascending=[False, True]).copy()
scratch_df3.head()

"""
Explanation: Sort by values of multiple variables and specify sort order
End of explanation
"""


scratch_df2.sort_index().head()

"""
Explanation: Sort by the value of the dataframe index
End of explanation
"""


# create a toy dataframe to join/merge onto scratch_df
scratch_df3 = scratch_df3.drop(['numeric1', 'numeric2'] , axis=1)
scratch_df3.columns = ['char3', 'char4']
scratch_df3.tail()

# default outer join on matching indices
# this will create 2000 row × 6 column dataset because indices are not in identical order
scratch_df4 = pd.concat([scratch_df, scratch_df3])
scratch_df4

# outer join on matching columns
# axis=1 specificies to join on matching columns
scratch_df5 = pd.concat([scratch_df, scratch_df3], axis=1)
scratch_df5.head()

scratch_df5.shape

# append
scratch_df6 = scratch_df.append(scratch_df)
scratch_df6.shape

"""
Explanation: 8. Adding data to the dataframe

Pandas concat() supports numerous types of joins and merges
Pandas merge() supports joins and merges using more SQL-like syntax 
i.e. merge(left, right, on=)
Pandas append() supports stacking dataframes top-to-bottom
End of explanation
"""


scratch_df.equals(scratch_df)

scratch_df.equals(scratch_df.sort_values(by='char1'))

scratch_df.equals(scratch_df2)

"""
Explanation: 9. Comparing dataframes

Use Pandas equals() to compare dataframes
Row order is not ignored
End of explanation
"""


scratch_df.mean()

scratch_df.mode()

scratch_df.describe()

"""
Explanation: 10. Summarizing dataframes
Pandas offers several straightforward summarization functions
End of explanation
"""


# use summary function size() on groups created by groupby()
counts = scratch_df.groupby('char1').size()
plt.figure()
counts.plot.bar(title='Frequency of char1 values (Histogram of char1)')

# groupby the values of more than one variable
group_means = scratch_df.groupby(['char1', 'char2']).mean()
group_means

"""
Explanation: 11. By group processing
Use Pandas groupby() to create groups for subsequent processing
End of explanation
"""


# Pandas .T performs a transpose
scratch_df.T.iloc[:, 0:5]

"""
Explanation: 12. Transposing a table

Transposing a matrix simply switches row and columns values
Transposing a data.frame or dplyr table is more complex because of metadata associated with variable names and row indices
End of explanation
"""


# export to csv
scratch_df.to_csv('scratch.csv')

# import from csv 
scratch_df7 = pd.read_csv('scratch.csv')

"""
Explanation: Often, instead of simply transposing, a data set will need to be reformatted in a melt/stack - column split - cast action described in Hadley Wickham's Tidy Data:
https://www.jstatsoft.org/article/view/v059i10
See the stack and unstack methods for Pandas dataframes
Exporting and importing a dataframe

Many to_* methods available for exporting dataframes to other formats
Many read_* methods available for creating dataframes from other formats
End of explanation
"""


