import os
import datetime
import time
import tarfile

DATA_FILE = '../data/menus-2015_07_16_07_01_00_data.tgz'

tar = tarfile.open(DATA_FILE)

for tf in tar.getmembers():
    print('Name: {0} \t Last Modified: {1}'.format(tf.name, time.ctime(tf.mtime)))

DATA_DIR = '../data/nypl_menus'
tar.extractall(path=DATA_DIR)

for f in os.listdir(DATA_DIR):
    if f.endswith('csv'):
        if os.path.isfile(os.path.join(DATA_DIR, f)) == True:
            print('{0} … \u2713'.format(f))

tar.close()

"""
Explanation: Power Tools: Python Pandas
To demonstrate the usefulness of the pandas Python library, we'll walk through a real life use case. An extract-transform-load pipeline we built for working with the NYPL menus data:
A Repeatable Extract-Tranform-Load Pipeline for NYPL Menus Data
Created: 17 October 2014
Updated: 28 October 2014, 19 November 2014, 3 December 2014
Authors: Trevor Muñoz and Katie Rawson
&nbsp;
Acquiring Data (Extract)
For the purpose of this exercise, we'll grab a copy from our local data folder
End of explanation
"""


import re
import pandas as pd

LATEST_DISH_DATA_DF = pd.DataFrame.from_csv(os.path.join(DATA_DIR, 'Dish.csv'), 
                                            index_col='id')
LATEST_ITEM_DATA_DF = pd.DataFrame.from_csv(os.path.join(DATA_DIR, 'MenuItem.csv'), 
                                            index_col='dish_id')
LATEST_PAGE_DATA_DF = pd.DataFrame.from_csv(os.path.join(DATA_DIR, 'MenuPage.csv'), 
                                            index_col='id')
LATEST_MENU_DATA_DF = pd.DataFrame.from_csv(os.path.join(DATA_DIR, 'Menu.csv'),
                                             index_col='id')

"""
Explanation: Working with Data in DataFrames (Tranform)
&nbsp;
End of explanation
"""


NULL_APPEARANCES = LATEST_DISH_DATA_DF[LATEST_DISH_DATA_DF.times_appeared == 0]

print('Data set contains {0} dishes that appear 0 times …'.format(
    len(NULL_APPEARANCES))
)

NON_NULL_DISH_DATA_DF = LATEST_DISH_DATA_DF[LATEST_DISH_DATA_DF.times_appeared != 0]

discarded_columns = [n for n in NON_NULL_DISH_DATA_DF.columns if n not in 
                     ['name', 'menus_appeared', 'times_appeared']]

print('Discarding columns from Dish.csv …')
for discard in discarded_columns:
    print('{0} … removed'.format(discard))

TRIMMED_DISH_DATA_DF = NON_NULL_DISH_DATA_DF[['name', 'menus_appeared', 'times_appeared']]

print('Dish.csv contains {0} potentially-unique dish names before any normalization'.
                     format(TRIMMED_DISH_DATA_DF.name.nunique()))

def normalize_names(obj):
    '''
    Take a name as a string, converts the string
    to lowercase, strips whitespace from beginning
    and end, normalizes multiple internal whitespace
    characters to a single space. E.g.:
    
    normalize_names('Chicken gumbo ') = 'chicken gumbo'
    
    '''
    tokens = obj.strip().lower().split()
    result = ' '.join(filter(None, tokens))
    return result

TRIMMED_DISH_DATA_DF['normalized_name'] = TRIMMED_DISH_DATA_DF.name.map(normalize_names)

print(
    'Dish.csv contains {0} potentially-unique dish names after normalizing whitespace and punctuation'
    .format(TRIMMED_DISH_DATA_DF.normalized_name.nunique())
)

def fingerprint(obj):
    """
    A modified version of the fingerprint clustering algorithm implemented by Open Refine.
    See https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth
    This does not normalize to ASCII characters since diacritics may be significant in this dataset
    """
    alphanumeric_tokens = filter(None, re.split('\W', obj))
    seen = set()
    seen_add = seen.add
    deduped = sorted([i for i in alphanumeric_tokens if i not in seen and not seen_add(i)])
    fingerprint = ' '.join(deduped)
    
    return fingerprint

TRIMMED_DISH_DATA_DF['fingerprint'] = TRIMMED_DISH_DATA_DF.normalized_name.map(fingerprint)

print(
    'Dish.csv contains {0} unique fingerprint values'
    .format(TRIMMED_DISH_DATA_DF.fingerprint.nunique())
)

TRIMMED_DISH_DATA_DF.head()

"""
Explanation: Dish.csv
End of explanation
"""


discarded_columns2 = [n for n in LATEST_ITEM_DATA_DF.columns if n not in 
                      ['id', 'menu_page_id', 'xpos', 'ypos']]

print('Discarding columns from MenuItem.csv …')
for discard2 in discarded_columns2:
    print('{0} … removed'.format(discard2))

TRIMMED_ITEM_DATA_DF = LATEST_ITEM_DATA_DF[['id', 'menu_page_id', 'xpos', 'ypos']]

TRIMMED_ITEM_DATA_DF.head()

"""
Explanation: MenuItem.csv
End of explanation
"""


LATEST_PAGE_DATA_DF.head()

LATEST_PAGE_DATA_DF[['full_height', 'full_width']].astype(int, raise_on_error=False)

"""
Explanation: MenuPage.csv
End of explanation
"""


LATEST_MENU_DATA_DF.columns

discarded_columns3 = [n for n in LATEST_MENU_DATA_DF.columns if n not in 
                      ['sponsor', 'location', 'date', 'page_count', 'dish_count']]

pipeline_logger.info('Discarding columns from Menu.csv …')
for discard3 in discarded_columns3:
    pipeline_logger.info('{0} … removed'.format(discard3))

TRIMMED_MENU_DATA_DF = LATEST_MENU_DATA_DF[['sponsor', 'location', 'date',
                                            'page_count', 'dish_count']]

TRIMMED_MENU_DATA_DF.head()

"""
Explanation: Menu.csv
End of explanation
"""


MERGED_ITEM_PAGES_DF = pd.merge(TRIMMED_ITEM_DATA_DF, LATEST_PAGE_DATA_DF, 
                                left_on='menu_page_id', right_index=True, )

MERGED_ITEM_PAGES_DF.columns = ['item_id', 'menu_page_id', 'xpos', 'ypos', 
                                'menu_id', 'page_number', 
                                'image_id', 'full_height', 'full_width', 'uuid']

#MERGED_ITEM_PAGES_DF.head()

MERGED_ITEM_PAGES_MENUS_DF = pd.merge(TRIMMED_MENU_DATA_DF, MERGED_ITEM_PAGES_DF, 
                                      left_index=True, right_on='menu_id')

FULL_MERGE = pd.merge(MERGED_ITEM_PAGES_MENUS_DF, TRIMMED_DISH_DATA_DF, 
                      left_index=True, right_index=True)

FULL_MERGE.head()

FOR_JSON_OUTPUT = FULL_MERGE.reset_index()

FOR_JSON_OUTPUT.columns

renamed_columns = ['dish_id', 'menu_sponsor', 'menu_location', 'menu_date', 'menu_page_count', 
                   'menu_dish_count', 'item_id', 'menu_page_id', 'item_xpos', 'item_ypos', 
                   'menu_id', 'menu_page_number', 'image_id', 
                   'page_image_full_height', 'page_image_full_width', 'page_image_uuid', 'dish_name', 
                   'dish_menus_appeared', 'dish_times_appeared', 'dish_normalized_name', 'dish_name_fingerprint']

FOR_JSON_OUTPUT.columns = renamed_columns

FOR_JSON_OUTPUT[['menu_page_number', 'dish_id', 'item_id', 'menu_page_id', 'menu_id']].astype(int, raise_on_error=False)

FOR_JSON_OUTPUT['dish_uri']= FOR_JSON_OUTPUT.dish_id.map(lambda x: 'http://menus.nypl.org/dishes/{0}'.format(int(x)))

FOR_JSON_OUTPUT['item_uri']= FOR_JSON_OUTPUT.item_id.map(lambda x: 'http://menus.nypl.org/menu_items/{0}/edit'
                                               .format(int(x)))

FOR_JSON_OUTPUT['menu_page_uri'] = FOR_JSON_OUTPUT.menu_page_id.map(lambda x: 'http://menus.nypl.org/menu_pages/{0}'
                                                          .format(int(x)))

FOR_JSON_OUTPUT['menu_uri'] = FOR_JSON_OUTPUT.menu_id.map(lambda x:'http://menus.nypl.org/menus/{0}'
                                                .format(int(x)))

FOR_JSON_OUTPUT.head()

print('Generating JSON …')
FOR_JSON_OUTPUT.to_json(path_or_buf='../data/nypl_menus/menus_all.json', orient='index', force_ascii=False)

"""
Explanation: Merging DataFrames
End of explanation
"""


