import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import re
from nltk.tokenize import word_tokenize
from time import time
import pickle

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
df = pd.read_csv('/kaggle/input/scl-2021-ds/train.csv')
poi_street_df = df["POI/street"].str.split("/", n = 1, expand = True) 
df["POI"]= poi_street_df[0] 
df["street"]= poi_street_df[1] 
df1 = df.drop(["POI/street"], axis=1) 
df1
"""
## Data Cleaning

### 1. Let's fix discrepancies between `street` and `raw_address` labels.
"""
def find_first_index(ra_split, st_split):
    
    if len(st_split) <= len(ra_split): # new
    
        num_iter = len(ra_split) - len(st_split) + 1
        overlap_list = []

        for i in range(num_iter):
            window = ra_split[i: i+len(st_split)]
            overlap = list(set(window) & set(st_split))
            overlap_list.append(len(overlap))

        max_overlap = [e for e in range(len(overlap_list)) if overlap_list[e] == max(overlap_list)]
        if len(max_overlap) == 1:
            return max_overlap[0]

        else:
            count_list = []
            for idx in max_overlap:
                subset_ra = ra_split[idx: idx+len(st_split)]
                count = 0
                for e in range(len(subset_ra)):
                    if subset_ra[e] not in st_split[e]:
                        count += 0
                    else:
                        count += 1
                count_list.append(count)
            index = count_list.index(max(count_list))
            return max_overlap[index]
    else:
        return 0
def fix_street_errors(row):
    
    raw_add = row['raw_address']

    # If a street name is extracted...
    if row['street'] != "":

        raw_add_split = word_tokenize(raw_add)
        
        extr_street = row['street']
        extr_street_split = word_tokenize(extr_street)
        
        # If the extracted street is in the raw address as an entire string, good!
        if extr_street in raw_add:
            return raw_add
        
        # This is where there are discrepancies!
        else:
            index_in_ra = find_first_index(raw_add_split, extr_street_split)
            raw_add_split[index_in_ra: index_in_ra+len(extr_street_split)] = extr_street_split
            updated_raw_add = ' '.join(raw_add_split).replace(' ,', ',').replace(' .', '.').replace(' )', ')').replace(' (', '(').replace(' ?', '?')          
            return updated_raw_add
      
    # If a street name is originally an empty string, we just assume there's no error. 
    else:
        return raw_add
start = time()

df1['cleaned_raw_add'] = df1.apply(fix_street_errors, axis=1)

print("Executed in {} minutes.".format(round((time() - start)/60, 3)))

# Sanity checks
df1.loc[[69, 86, 117, 130, 135, 169], :]
"""
### 2. We also build a dictionary mapping the incorrect/incomplete street names in `raw_address` to the correct `street` labels. 
"""
def get_street_mapping_dict(row):
    
    raw_add = row['raw_address']

    # If a street name is extracted...
    if row['street'] != "":

        raw_add_split = word_tokenize(raw_add)
        
        extr_street = row['street']
        extr_street_split = word_tokenize(extr_street)
        
        # If the extracted street is in the raw address as an entire string, good!
        if extr_street in raw_add:
            return None
        
        # This is where there are discrepancies!
        else:
            index_in_ra = find_first_index(raw_add_split, extr_street_split)
            before = raw_add_split[index_in_ra: index_in_ra+len(extr_street_split)] 
            before = ' '.join(before)
            return before, extr_street
      
    # If a street name is originally an empty string, we just assume there's no error. 
    else:
        return None
start = time()

df1['street_mapping'] = df1.apply(get_street_mapping_dict, axis=1)

print("Executed in {} minutes.".format(round((time() - start)/60, 3)))

# Sanity checks
df1.loc[[69, 86, 117, 130, 135, 169], :]
# Create a separate dataframe containing only values in `street_mapping` columns, i.e., rows where changes occurred
df_with_street_mappings = df1[df1['street_mapping'].notnull()]

street_mapping_dict = dict()

# Create a mapping dictionary, where key is the truncated word/words and the value is the corresponding correct word/words
for row, col in df_with_street_mappings.iterrows():
    street_mapping_dict[col['street_mapping'][0]] = col['street_mapping'][1]
    
# How many street errors are there altogether?
print("Length of street mapping dictionary:", len(street_mapping_dict))
"""
### 3. Next, let's rectify discrepancies between `POI` and `raw_address` labels.
"""
def fix_poi_errors(row):
    
    raw_add = row['cleaned_raw_add']

    # If a POI name is extracted...
    if row['POI'] != "":

        raw_add_split = word_tokenize(raw_add)
        
        extr_poi = row['POI']
        extr_poi_split = word_tokenize(extr_poi)
        
        # If the extracted POI is in the raw address as an entire string, good!
        if extr_poi in raw_add:
            return raw_add
        
        # This is where there are discrepancies!
        else:
            index_in_ra = find_first_index(raw_add_split, extr_poi_split)
            raw_add_split[index_in_ra: index_in_ra+len(extr_poi_split)] = extr_poi_split
            updated_raw_add = ' '.join(raw_add_split).replace(' ,', ',').replace(' .', '.').replace(' )', ')').replace(' (', '(').replace(' ?', '?')          
            return updated_raw_add
      
    # If a POI name is originally an empty string, we just assume there's no error. 
    else:
        return raw_add
start = time()

df1['cleaned_raw_add_1'] = df1.apply(fix_poi_errors, axis=1)

print("Executed in {} minutes.".format(round((time() - start)/60, 3)))

# Sanity checks
df1.loc[[10, 11, 40, 110, 152, 157, 169], :]
"""
### 4. Likewise, we also build a dictionary mapping the incorrect/incomplete POI names in `raw_address` to the correct `POI` labels. 
"""
def get_poi_mapping_dict(row):
    
    raw_add = row['cleaned_raw_add']

    # If a POI name is extracted...
    if row['POI'] != "":

        raw_add_split = word_tokenize(raw_add)
        
        extr_poi = row['POI']
        extr_poi_split = word_tokenize(extr_poi)
        
        # If the extracted POI is in the raw address as an entire string, good!
        if extr_poi in raw_add:
            return None
        
        # This is where there are discrepancies!
        else:
            index_in_ra = find_first_index(raw_add_split, extr_poi_split)
            before = raw_add_split[index_in_ra: index_in_ra+len(extr_poi_split)] 
            before = ' '.join(before)
            return before, extr_poi
      
    # If a POI name is originally an empty string, we just assume there's no error. 
    else:
        return None
start = time()

df1['poi_mapping'] = df1.apply(get_poi_mapping_dict, axis=1)

print("Executed in {} minutes.".format(round((time() - start)/60, 3)))

# Sanity checks
df1.loc[[10, 11, 40, 110, 152, 157, 169], :]
# Create a separate dataframe containing only values in `poi_mapping` columns, i.e., rows where changes occurred
df_with_poi_mappings = df1[df1['poi_mapping'].notnull()]

poi_mapping_dict = dict()

# Create a mapping dictionary, where key is the truncated word/words and the value is the corresponding correct word/words
for row, col in df_with_poi_mappings.iterrows():
    poi_mapping_dict[col['poi_mapping'][0]] = col['poi_mapping'][1]
    
# How many POI errors are there altogether?
print("Length of POI mapping dictionary:", len(poi_mapping_dict))
"""
### 5. We've cleaned the raw address to make sure that it tallies with both the extracted `street` and `POI` labels. Now, let's tidy things up. 
"""
# Select required columns
cleaned_df = df1[['id','cleaned_raw_add_1', 'POI', 'street']]

# Rename columns
cleaned_df.columns = ['id', 'raw_address', 'POI', 'street']

# Sanity checks
cleaned_df.loc[[10, 11, 40, 69, 86, 110, 117, 130, 135, 152, 157, 169], :]
"""
### 6. Now that we have the two mapping dictionaries, let's check whether some of those truncated words exist in the test data, and if so, replace them with the correct labels. 
"""
# Load test dataset
test_df = pd.read_csv('/kaggle/input/scl-2021-ds/test.csv')

# Preview
test_df.head()
"""
It might be risky to replace single words like `par` in the test data as they may form part of a bigger word that is different from the intended word. For example, we should not replace "par" in the address "daya paru 43" with its value in the `street_mapping_dict`, "parigi". This is less likely the case if there are two or more words.

Let's try to filter out single words from the list.
"""
new_street_mapping_dict = {k: v for k, v in street_mapping_dict.items() if len(k.split()) > 1}
new_poi_mapping_dict = {k: v for k, v in poi_mapping_dict.items() if len(k.split()) > 1}

print("Length of street mapping dictionary after removing single words:", len(new_street_mapping_dict))
print("Length of POI mapping dictionary after removing single words:", len(new_poi_mapping_dict))
start = time()

# Replace truncated words in raw_address of test set with correct street labels
count1 = 0
for row, col in test_df.iterrows():
    for k, v in new_street_mapping_dict.items():
        if k in col['raw_address']:
            test_df.loc[row, 'raw_address'] = test_df.loc[row, 'raw_address'].replace(k, v)
            count1 += 1
            
print("Number of raw addresses updated due to errors in street labels:", count1)

# Replace truncated words in raw_address of test set with correct POI labels
count2 = 0
for row, col in test_df.iterrows():
    for k, v in new_poi_mapping_dict.items():
        if k in col['raw_address']:
            test_df.loc[row, 'raw_address'] = test_df.loc[row, 'raw_address'].replace(k, v)
            count2 += 1

print("Number of raw addresses updated due to errors in POI labels:", count2)

print("Executed in {} minutes.".format(round((time() - start)/60, 3)))

test_df
# 0
# s. par 53 sidanegara 4 cilacap tengah to be replaced with s. par
# Updated:  s. parman 53 sidanegara 4 cilacap tengah
# 1
# angg per, baloi indah kel. lubuk baja to be replaced with angg per
# Updated:  anggrek per, baloi indah kel. lubuk baja
# 2
# asma laun, mand imog, to be replaced with , man
# Updated:  asma laun, mangund imog,
# 3
# ud agung rej, raya nga sri wedari karanganyar to be replaced with raya nga
# Updated:  ud agung rej, raya ngawi- sri wedari karanganyar
# 5
# pem dos dapur ala perum gar no a 12 suka jaya sukarami to be replaced with perum gar
# Updated:  pem dos dapur ala perumahan gar no a 12 suka jaya sukarami
# 9
# raya won wonotunggal wonotunggal to be replaced with raya won
# Updated:  raya wonoso wonotunggal wonotunggal
# 10
# tebet timur tebet raya 92 rt 2 1 tebet to be replaced with bet raya
# Updated:  tebet timur tebetung raya 92 rt 2 1 tebet
# 14
# toko teddy raya pan jakat, to be replaced with raya pan
# Updated:  toko teddy raya pandan jakat,
# 14
# toko teddy raya pan jakat, to be replaced with raya pan jakat
# Updated:  toko teddy raya pandan jakat,
# 36
# m. t. haryono, no 11 bank neg indonesia kali rejo ungaran timur to be replaced with bank negara indonesia
# Updated:  m. t. haryono, no 11 bank negara indonesiaesia kali rejo ungaran timur
# 36
# m. t. haryono, no 11 bank neg indonesia kali rejo ungaran timur to be replaced with bank negara indonesia
# Updated:  m. t. haryono, no 11 bank negara indonesiaesia kali rejo ungaran timur
# 47
# rezi, pasar raya to be replaced with pasar campor
# Updated:  rezi, pasar campor
# 52
# mas nurul huda badang ngoro to be replaced with masjid nurul huda
# Updated:  masjid nurul huda badang ngoro
# 86
# kateguhan roti bakar bandung pak budi, jenderal sudirman, to be replaced with roti bakar bandung
# Updated:  kateguhan roti bakar bandungung pak budi, jenderal sudirman,
# 92
# kenc utama ii kembangan selatan 7 kembangan to be replaced with kencana utama
# Updated:  kencana utama ii kembangan selatan 7 kembangan
"""
### 7. Save the cleaned-up train and test datasets, as well as the two mapping dictionaries.
"""
# Save the cleaned train dataset
cleaned_df.to_csv('cleaned_train.csv', index=False)

# Save the cleaned test dataset
test_df.to_csv('cleaned_test.csv', index=False)
# Save the dictionaries
s_file = open("street_mapping_dict.pkl", "wb")
pickle.dump(street_mapping_dict, s_file)
s_file.close()

p_file = open("poi_mapping_dict.pkl", "wb")
pickle.dump(street_mapping_dict, p_file)
p_file.close()
"""
### Note: Run the following codes to load the two dictionaries
"""
# s_file = open("street_mapping_dict.pkl", "rb")
# street_mapping_dict = pickle.load(s_file)
# print(street_mapping_dict)

# p_file = open("poi_mapping_dict.pkl", "rb")
# poi_mapping_dict = pickle.load(p_file)
# print(poi_mapping_dict)