import pandas as pd

"""
Explanation: Pandas for managing Datasets
End of explanation
"""


df = pd.DataFrame()
df

"""
Explanation: Create an empty dataframe for adding in datasets
End of explanation
"""


for chunk in pd.read_csv('data/ext_lt_invcur.tsv', sep='\t', chunksize=100):
    df = pd.concat([df, chunk])

df.head()

"""
Explanation: Importing a large dataset in chunks of 100 bytes
End of explanation
"""


for chunk in pd.read_csv('data/ext_lt_invcur.tsv', sep='\t', chunksize=100):
    # separate out all the data rows for column index 0
    data_rows = [row for row in chunk.ix[:,0].str.split(',')]
    
    # show me column indexed 0 and split on ","
    data_cols = [col.split('\\')[0] for col in chunk.columns[0].split(',')]
    
    # create new data frame with these new rows and columns
    clean_df = pd.DataFrame(data_rows, columns=data_cols)
    new_df = pd.concat([clean_df, chunk.drop(chunk.columns[0], axis=1)], 
                       axis=1)
    
    # sanity check for top 5 rows
    print new_df.head()
    break
    df = pd.concat([df, chunk])

for chunk in pd.read_csv('data/ext_lt_invcur.tsv', sep='\t', chunksize=100):
    # separate out all the data rows for column index 0
    data_rows = [row for row in chunk.ix[:,0].str.split(',')]
    
    # show me column indexed 0 and split on ","
    data_cols = [col.split('\\')[0] for col in chunk.columns[0].split(',')]
    
    # create new data frame with these new rows and columns
    clean_df = pd.DataFrame(data_rows, columns=data_cols)
    new_df = pd.concat([clean_df, chunk.drop(chunk.columns[0], axis=1)], 
                       axis=1)
    
    df = pd.concat([df, new_df])

"""
Explanation: Notice that column 2 looks like can be further transformed during the loading phase.
End of explanation
"""


