# Import py_entitymatching package
import py_entitymatching as em
import os
import pandas as pd

"""
Explanation: Introduction
This IPython notebook illustrates how to use multiple blockers and combine the results.
First, we need to import py_entitymatching package and other libraries as follows:
End of explanation
"""


# Get the datasets directory
datasets_dir = em.get_install_path() + os.sep + 'datasets'

# Get the paths of the input tables
path_A = datasets_dir + os.sep + 'person_table_A.csv'
path_B = datasets_dir + os.sep + 'person_table_B.csv'

# Read the CSV files and set 'ID' as the key attribute
A = em.read_csv_metadata(path_A, key='ID')
B = em.read_csv_metadata(path_B, key='ID')

"""
Explanation: Then, read the (sample) input tables for blocking purposes.
End of explanation
"""


#Blocking plan :

#A, B --overlap blocking--> candset --attr-equiv-block--> candset  |
#                                                                  | 
#A, B ------------rule-based-blocking--------------------> candset |----union--->candset
#                                                                  |
#A, B -----------black-box-blocking----------------------> candset |


# Overlap blocking over input tables
ob = em.OverlapBlocker()
# block using name
C = ob.block_tables(A, B, 'name', 'name', word_level=True, overlap_size=1, 
                    l_output_attrs=['name', 'birth_year'], 
                    r_output_attrs=['name', 'birth_year'],
                    show_progress=False)
C

# Overlap blocking over input tables
ob = em.OverlapBlocker()
# block using name
C = ob.block_tables(A, B, 'name', 'name', word_level=True, overlap_size=1, 
                    l_output_attrs=['name', 'birth_year'], 
                    r_output_attrs=['name', 'birth_year'],
                    show_progress=False)
C

# Attribute equivalence blocking: block C using birth_year
ab = em.AttrEquivalenceBlocker()
D = ab.block_candset(C, 'birth_year', 'birth_year', show_progress=False)

D

# Rule-based blocking over input tables
# first get features that can be used
feature_table = em.get_features_for_blocking(A, B, validate_inferred_attr_types=False)

# Create rule-based blocker
rb = em.RuleBasedBlocker()
# Add rule : block tuples if name_name_lev(ltuple, rtuple) < 0.4
rb.add_rule(['name_name_lev_sim(ltuple, rtuple) < 0.4'], feature_table)

E = rb.block_tables(A, B, l_output_attrs=['name'], r_output_attrs=['name'], show_progress=False)

# Apply black box blocker
# Create black box blocker
bb = em.BlackBoxBlocker()

# define a black box function. 
# The blocker function should drop tuple pairs whose last name do not match
# The function has to do the following steps
#  1) Get name attributes from each of the tuples
#  2) Split name attribute to get last name
#  3) if last names donot match return True

def my_function(x, y):
    # x, y will be of type pandas series
    
    # get name attribute
    x_name = x['name']
    y_name = y['name']
    # get last names
    x_name = x_name.split(' ')[1]
    y_name = y_name.split(' ')[1]
    # check if last names match
    if x_name != y_name:
        return True
    else:
        return False

bb.set_black_box_function(my_function)

F = bb.block_tables(A, B, l_output_attrs=['name'], r_output_attrs=['name'], show_progress=False)

F

# Combine all the blocker outputs
G = em.combine_blocker_outputs_via_union([D, E, F])

G

em.show_properties(G)

"""
Explanation: Combining Multiple Blockers
End of explanation
"""


