import pandas as pd
import numpy as np
import re
from scipy import sparse as sparse

# SK-learn libraries for feature extraction from text.
from sklearn.feature_extraction.text import *

data_dir = "/Users/seddont/Dropbox/Tom/MIDS/W209_work/Tom_project/"
code_dir = "/Users/seddont/Dropbox/Tom/MIDS/W209_work/w209finalproject/"

"""
Explanation: Processing the open food databse to extract a dataset to use for the visualization.
End of explanation
"""


# Get sample of the full database to understand what columns we want
smp = pd.read_csv(data_dir+"en.openfoodfacts.org.products.csv", sep = "\t", nrows = 100)

for c in smp.columns:
    print(c)

# Specify what columns we need for the visualization.  For speed purposes going to
# remove any we don't really need

wanted_cols = ['code', 'creator', 'product_name', 'brands', 'brands_tags', 'serving_size',
       'serving_size', 'energy_100g', 'fat_100g', 'cholesterol_100g',
       'carbohydrates_100g', 'sugars_100g', 'fiber_100g', 'proteins_100g', 'sodium_100g']

# Create a list of columns to drop to check it worked ok
drop_cols = [c for c in smp.columns if c not in wanted_cols]
print(drop_cols)

# Pull in full dataset, only the columns we want
df = pd.read_csv(data_dir+"en.openfoodfacts.org.products.csv", sep = "\t")

# Drop unwanted columns
df.drop(drop_cols, axis = 1, inplace = True)

# Take a quick look
df

# Drop all rows that are not from the usda ndb import
df = df[df.creator == "usda-ndb-import"]

# Drop all rows where Brands == Nan as we can't really identify those products
df = df[df.brands.notnull()]

df

"""
Explanation: Working from the full database, because the usda_imports_filtered.csv file in the shared drive does not have brand information, which might be useful for displaying.
End of explanation
"""


df.code.apply(str)

# Add an N in front of the number string
df.code = "N"+df.code.astype(str)

"""
Explanation: The code column can be problematic as it's a long number that can be truncated to a floating point representation when manipulated by certain programs.  Going to convert it to append a character at the start to it will be read unambiguously as a string.
First convert the code column to a string
End of explanation
"""


df.isnull().sum(axis = 0)

"""
Explanation: Quick check for NA values
End of explanation
"""


df = df[df.product_name.notnull()]

"""
Explanation: Need to get rid of rows with null product_name as we will be using that later for display etc.
End of explanation
"""


df

# Get all the non-null product name fields
name_data = df.product_name

# Pass them to the Count Vectorizer
vectorizer = CountVectorizer()
wv = vectorizer.fit_transform(name_data)

# Get some basic stats
print("Size of vocabulary:", wv.shape[1],"words")
print("Average non-zero entries per example:%5.2f" % (1.0*wv.nnz/wv.shape[0]))

"""
Explanation: Now going to want to try and find similar products to a specific item.  In the demo, did this by just pulling items with certain text in the name and then doing some hand sorting.  Can't scale that approach.  Going to try to do it based on a combination of words in the name and the ingredients.
Try out the approach of using CountVectorizer on the product_name field to see how it does.
End of explanation
"""


a = vectorizer.get_feature_names()
print("First feature string:", sorted(a)[0])
print("Last feature string:", sorted(a)[len(a)-1])

"""
Explanation: Get the extracted words
End of explanation
"""


fn = vectorizer.get_feature_names()
wc = wv.sum(axis = 0)
word_frame = pd.DataFrame({"word": fn, "count":np.ravel(wc)}) 

word_frame.sort_values(by = ["count"], ascending = False)

"""
Explanation: Count occurences by word
End of explanation
"""


# Get all the non-null product name fields
name_data = df.product_name[df.product_name.notnull()]

# Pass them to the Count Vectorizer
vectorizer = CountVectorizer(analyzer = "word", ngram_range = (3,3))
wv = vectorizer.fit_transform(name_data)

# Get some basic stats
print("Size of vocabulary:", wv.shape[1],"words")
print("Average non-zero entries per example:%5.2f" % (1.0*wv.nnz/wv.shape[0]))

fn = vectorizer.get_feature_names()
wc = wv.sum(axis = 0)

word_frame = pd.DataFrame({"word": fn, "count":np.ravel(wc)}) 

word_frame.sort_values(by = ["count"], ascending = False)

"""
Explanation: Ok -- challenge here is that many of the words are descriptive adjectives vs nouns.  Even the ones that are nouns are going to be hard to separate e.g. rice could be rice crackers, chicken and rice, etc.
Wonder if I can get anywhere with bigrams or trigrams...try again
End of explanation
"""


word_frame.to_csv(data_dir+"trigrams_uncoded.csv")

tri_w = pd.read_csv(code_dir+"trigrams_wanted_v2.csv")

tri_w

"""
Explanation: So some kind of bigram and trigram approach might be scalable here.  But don't really have a lot of time to perfect that.  However, could use these trigrams to expand the simple 'demo cat' approach by picking only certain bigrams and then using the simple algorithm from the demo approach to find a range of other products with that bigram.
Going to write out the trigrams, do some hand coding of which ones we want to use, then bring the results back in
End of explanation
"""


tri_w = tri_w[tri_w.wanted.notnull()]
tri_w

"""
Explanation: Can drop all the rows where wanted is NaN
End of explanation
"""


# Initialize demo category with None
df["demo_cat"] = "None"

# loop over each trigram
for trigram in tri_w.word:
 
    # get the index of the correct column for that trigram in the vectorized output
    wv_index = fn.index(trigram)
    
    # Get locations of matches and convert to a dense representation for indexing
    matches = wv[:,wv_index] == 1    
    matches = np.ravel(sparse.csr_matrix.todense(matches))
      
    # Set the 'demo_cat' field to that trigram value
    df.loc[matches, ["demo_cat"]] = trigram

df[df.demo_cat != "None"]

"""
Explanation: So now got an identifier to use to append wanted categories from
Want to append the trigram as the 'demo category' for any product which contains that trigram.  It is possible that a product will fit into multiple trigrams, in which case I will choose to put it in the lowest total count trigram category.  Going to do this by looping through the wanted trigrams from highest to lowest and updating the product's category as needed.
End of explanation
"""


# loop over each trigram
enough_variance = []

for trigram in tri_w.word:
    
    fat_var = df[df.demo_cat == trigram]["fat_100g"].var()
    
    if fat_var > 2:
        enough_variance.append(trigram)

print(enough_variance)
print(len(enough_variance))

"""
Explanation: Some categories have very little variation in them.  Going to check for that and drop them.
End of explanation
"""


# What we want to get variation on
pick_factors = ['fat_100g', 'sugars_100g', 'proteins_100g', 'sodium_100g']

# Points we want to pick (percentiles).  Can tune this to get more or fewer picks.

pick_percentiles = [0.1, 0.5, 0.9]
# pick_percentiles = [0, 0.25, 0.5, 0.75, 1.0]

demo_picks = []

# loop over each trigram that has enough variance in it

for cat in enough_variance:
    
    # first get all the items containing the cat word
    catf = df[df["demo_cat"] == cat]
        
    # Identify what rank each product is in that category, for each main factor
    for p in pick_factors:
        
        catf[p + "_rank"] = catf[p].rank(method = "first")
        
        # Select products at chosen percentiles on each
        
        high = catf[p + "_rank"].max()
        
        pick_index = [max(1, round(n * high)) for n in pick_percentiles]
        
        # add codes for those products
        demo_picks.extend(catf[catf[p+"_rank"].isin(pick_index)].code)
        
    
demo_df = df[df.code.isin(demo_picks)]
demo_df

demo_df[demo_df.demo_cat == "pasta enriched macaroni"]

df[df.demo_cat == "pasta enriched macaroni"]

"""
Explanation: Now got a category applied to a subset of the database.  Can run the same code as before to use that category to create a subset of recommendations for each category picked...
End of explanation
"""


def truncate_brand(s):
    if type(s) != str:
        return ""
    elif s.find(",") == -1:
        return s
    else:
        return s[:s.find(",")]

print(truncate_brand("Kroger, The Kroger Co."))
print(truncate_brand("Roundy's"))

demo_df.dtypes

demo_df["short_brand"] = demo_df.brands.apply(truncate_brand)

demo_df

"""
Explanation: Now need to try some processing on product name as many will be too similar and uninformative without brand name.
End of explanation
"""


demo_df["orig_product_name"] = demo_df.product_name

demo_df["new_product_name"] = demo_df.short_brand + " " + demo_df.product_name

demo_df.product_name = demo_df.new_product_name

"""
Explanation: Now going to save original product name and replace with a combination of that plus the brand
End of explanation
"""


demo_df["hos"] = 0
demo_df["image_url"] = None

"""
Explanation: Need to add blanks for the other columns which only exist in the hand-curated demo data.
End of explanation
"""


orig_demo = pd.read_csv(data_dir+"demo_food_data_latest.csv")

# Specify what columns we need to keep to match the regular dataframe above.

wanted_cols = ['code', 'creator', 'hos', 'image_url', 'product_name', 'brands', 'brands_tags', 'serving_size',
       'serving_size', 'energy_100g', 'fat_100g', 'cholesterol_100g',
       'carbohydrates_100g', 'sugars_100g', 'fiber_100g', 'proteins_100g', 'sodium_100g',
        'demo_cat']

# Create a list of columns to drop to check it worked ok
drop_cols = [c for c in orig_demo.columns if c not in wanted_cols]
print(drop_cols)

# Drop unwanted columns in orig demo
orig_demo.drop(drop_cols, axis = 1, inplace = True)
orig_demo

# Drop unwanted columns in demo_df
# Create a list of columns to drop to check it worked ok
drop_cols = [c for c in demo_df.columns if c not in wanted_cols]
print(drop_cols)
demo_df.drop(drop_cols, axis = 1, inplace = True)
demo_df

missing_cols = [col for col in demo_df.columns if col not in orig_demo.columns]
print("Missing columns", missing_cols)

"""
Explanation: Now want to append the original demo data products and their hall of shame status so we always have them in too.
End of explanation
"""


finished = demo_df.append(orig_demo)
finished

"""
Explanation: Now want to append the original demo data to the other data
End of explanation
"""


# finished.to_csv(code_dir+"demo_food_data_final.csv", index = False)
finished.to_csv(code_dir+"demo_food_data_final2.csv", index = False)

"""
Explanation: Can now write this out to file.
End of explanation
"""


