# Import Pandas & NumPy
import pandas as pd
import numpy as np

# Create a tiny dataset, as a list of tuples
name = ('Oslo','Copenhaguen','Helsinki','Stockholm','Reykjavik')
pop = ( 647676, 583348, 626305, 917297, 121822 )
area = ( 480.76, 86.20, 715.49, 188.0,  273 )
data = [ (1000+i,n,p,s) for i, (n,p,s) in enumerate(zip(name,pop,area)) ]        

"""
Explanation: Basic Pandas DataFrame operations
A few first steps with Pandas.
We'll create a small dataframe, access its elements, and enlarge it by adding new columns and rows
End of explanation
"""


# Create the dataframe from the list of tuples. We need to add the names of the columns, plus
# the column(s) we want to be used as row index
df = pd.DataFrame.from_records( data=data, columns=('id','name','population','area'), index=['id'] )

"""
Explanation: 1 Creating a DataFrame
End of explanation
"""


print df

# See the options we've got for data formatting
pd.describe_option('display')

"""
Explanation: Let's view the dataframe. We can print it:
End of explanation
"""


df

# Check dataframe dimensions
print df.shape
# Check dataframe components
print df.index
print df.columns

"""
Explanation: Or we can just show it, and it will be nicely formatted.
Note the double header: the second header row is for the column(s) forming the DataFrame index.
End of explanation
"""


df['name']

# Or also
df.name

"""
Explanation: 2 Fetching columns
End of explanation
"""


df[ ['name','population'] ]

"""
Explanation: We can also get more than one column. 
These operations create and return a new DataFrame
End of explanation
"""


df.loc[:,['name','population']]

"""
Explanation: Same thing, but this time we get a reference to the original DataFrame by using a locator operator (see next section)
End of explanation
"""


# One row, using the index. Note that in this case our row index is the 'id' column
df.loc[1000]

# Two rows
df.loc[1002:1003]

# Two rows, but only selected columns
df.loc[1002:1003,'name':'population']

"""
Explanation: 3 Accessing DataFrame contents
There are several ways  of accessing the elements contained in a DataFrame
3.1 By label
We can acccess rows and columns by using labels, i.e. the index for the rows and/or columns, using the loc locator.
End of explanation
"""


# Get the first row
df.iloc[0]

# Get the last row
df.iloc[-1]

"""
Explanation: 3.2 By position
And we can also select row/columns by their position  using the iloc locator.
End of explanation
"""


df[df.area<200]

df[ (df.area<200) & (df.population>600000) ]

# This variant returns the same size as the original dataframe, but fills only the rows that satisty the condition
df.where( df.area<200 )

"""
Explanation: 3.3 By boolean selection
Another possibility is to use a logical expression to create a boolean matrix, and index with it, selecting the rows that satisfy the expression
End of explanation
"""


df.sample(n=3)

"""
Explanation: 3.4 Random sample
End of explanation
"""


# We create a new column by combining data from other columns
df.loc[:,'density'] = df.loc[:,'population']/df.loc[:,'area']

df.head()

"""
Explanation: 4 Augmenting a DataFrame
Let's take the existing dataframe and enlarge it by adding new rows/columns
4.1 Adding a column
End of explanation
"""


df2 = df.assign( density2 = lambda x : x.population/x.area )
df2.head()

"""
Explanation: Another way of doing it is to use the assign() method. Ir returns a new DataFrame with the additions.
End of explanation
"""


# Find the next id to insert
next = df.tail(1).index.values[0] + 1

# Define new rows. This time, for a change, we'll be using a dict of lists as input data
name = ('Tallinn', 'Riga', 'Vilnius')
pop = ( 439286, 641007, 542664 )
size = ( 159.2, 304, 401 )
data2 = { 'id' : range(next,next+len(name)),
          'name' : name, 
          'population' : pop, 
          'area' : size  }
#data = [ {'id':next+i, 'name':n, 'population': p, size:'s' } 
#         for i, (n,p,s) in enumerate(zip(name,pop,size)) ]        

# Create a dataframe from the dict of lists
df2 = pd.DataFrame( data2 )
# Set the column(s) to be used as the row index in this new dataframe
df2.set_index( 'id', inplace=True )
#df2 = pd.DataFrame.from_dict( data )
#df.append( data, ignore_index=True)


df2

# Now append this set of rows to the original one
df = df.append(df2)
df

# Find the rows having a missing density value. Obviously they will be the just added ones
missing = df[ np.isnan(df.density) ].index

df.loc[missing]

"""
Explanation: 4.2 Adding rows
End of explanation
"""


df.loc[missing].density = df.loc[missing].population/df.loc[missing].area

df.loc[missing]

"""
Explanation: Now let's add the missing densities. First naive attempt:
End of explanation
"""


df.loc[missing,'density'] = df.loc[missing,'population']/df.loc[missing,'area']

"""
Explanation: It didn't work. Why? Because we are selecting in two steps:
* first we get the rows df.loc[missing]
* and then we get the column of those rows df.loc[missing].population
This is chained indexing. And it fails when using it for assignment
So let's try again, using a single-step indexing:
End of explanation
"""


df.loc[missing].density

"""
Explanation: This time it works:
End of explanation
"""


