import numpy as np

# Generating a random array
X = np.random.random((3, 5))  # a 3 x 5 array

print(X)

"""
Explanation: Data Manipulation with Numpy and Pandas
Handling with large data is easy in Python. In the simplest way using arrays. However, they are pretty slow. Numpy and Panda are two great libraries for dealing with datasets. Numpy isused for homogenous n-dimensional data (matrices). Pandas is used for heterogenous tables (CSV, MS Excel tables). Pandas is internally based on Numpy, too. See http://scipy-lectures.github.io/ for a more detailed lesson.
End of explanation
"""


# get a single element
X[0, 0]

# get a row
X[1]

# get a column
X[:, 1]

# Transposing an array
X.T

print(X.shape)
print(X.reshape(5, 3)) #change the layout of the matrix

# indexing by an array of integers (fancy indexing)
indices = np.array([3, 1, 0])
print(indices)
X[:, indices]  

"""
Explanation: Accessing elements
End of explanation
"""


X

X.shape

np.sum(X, axis=1) # 1...columns

np.max(X, axis=0) # 0...rows

"""
Explanation: Operations along an axis
End of explanation
"""


import numpy as np
import pandas as pd

#use a standard dataset of heterogenous data
cars = pd.read_csv('data/mtcars.csv')
cars.head()

#list all columns
cars.columns

#we want to use the car as the "primary key" of a row
cars.index = cars.pop('car')
cars.head()

#describe our dataset
cars.describe()

cars.sort_index(inplace=True)
cars.head()

cars.sort_values('mpg').head(15)

cars.sort_values('hp', ascending=False).head()

"""
Explanation: A quick-ish introduction to Pandas
based on http://pandas.pydata.org/pandas-docs/stable/10min.html
End of explanation
"""


#single column
cars['mpg']
#depending on the name also cars.mpg works

#or a slice of rows
cars[2:5]

#by label = primary key
cars.loc['Fiat 128':'Lotus Europa']

#selection by position
cars.iloc[3]

cars.iloc[3:5, 0:2]

cars[cars.cyl > 6] # more than 6 cylinders

"""
Explanation: Selection
Note While many of the NumPy access methods work on DataFrames, use the pandas-specific data access methods, .at, .iat, .loc, .iloc and .ix.
See the Indexing section and below.
End of explanation
"""


cars_na = pd.read_csv('data/mtcars_with_nas.csv')

cars_na.isnull().head(4)

#fill with a default value
cars_na.fillna(0).head(4)

#or drop the rows

print(cars_na.shape)
#drop rows with na values
print(cars_na.dropna().shape)
#drop columns with na values
print(cars_na.dropna(axis=1).shape)

#see also http://pandas.pydata.org/pandas-docs/stable/missing_data.html

"""
Explanation: Missing Data
End of explanation
"""


#stats
cars.mean()

cars.mean(axis=1)

#grouping
cars.groupby('cyl').mean()

#grouping different aggregation methods
cars.groupby('cyl').agg({ 'mpg': 'mean', 'qsec': 'min'})

"""
Explanation: Statistics
besides .describe() there are plenty of other staticial measures and aggregation methods in Pandas/Numpy
End of explanation
"""


#loading gapminder data (taken from https://github.com/jennybc/gapminder)
# file located at 'data/gapminder-unfiltered.tsv' it uses tabular character as separator
# use the first column as index


#what are the columns of this dataset?


#what is the maximal year contained?


#just select all data of the year 2007


#locate Austria and print it


#list the top 10 countries by life expectancy (lifeExp)


#what is the total population (pop) per continent


"""
Explanation: TASKS

Now there are a series of simple tasks
End of explanation
"""


