# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Vertex Smoothing
#
# Smoothing out or smoothing a vertex ***w*** with regards to the pair of edges (e1, e2) incident on ***w***, removes both edges containing ***w*** and replaces (e1, e2) with a new edge that connects the other endpoints of the pair.
#
# For example, the simple connected graph with two edges, e1 {u,w} and e2 {w,v}:
# ![alt text](https://upload.wikimedia.org/wikipedia/commons/6/6f/Graph_subdivision_step2.svg "Graph 1")
#
# has a vertex (namely w) that can be smoothed away, resulting in:
# ![alt text](https://upload.wikimedia.org/wikipedia/commons/1/15/Graph_subdivision_step1.svg "Graph 1")
#

# %load_ext autoreload
# %autoreload 2

from graphy.smoothing.vertexsmoothing import vertex_smoothing

# ## Setup

# + colab={} colab_type="code" id="dwinSYBfkmi4"
from pyspark.sql import SparkSession
spark = SparkSession.builder.master("local[*]").getOrCreate()

# + colab={"base_uri": "https://localhost:8080/", "height": 222} colab_type="code" id="wleyVpxMkpUR" outputId="45e2a9a7-1d99-4cfe-b6d8-d31a7979c4a0"
spark


# + [markdown] colab_type="text" id="J1STiXwxQo5l"
# ## Utility Functions

# + colab={} colab_type="code" id="Dz-I1fqyWEG4"
# Utils functions ============================
# TUPLE

def source_getter_tuple(x):
    return x[0]

def target_getter_tuple(x):
    return x[2]

def link_getter_tuple(x):
    return x[1]

def obj_creator_tuple(s, l, t):
    return (s, l, t)


# DICT
def source_getter_dict(x):
    return x['SOURCE']

def target_getter_dict(x):
    return x['TARGET']

def link_getter_dict(x):
    return x['LINK']

def obj_creator_dict(s, l, t):
    return {'SOURCE': s, 'LINK': l, 'TARGET': t}


# -

# Some example of usage...

# + colab={"base_uri": "https://localhost:8080/", "height": 271} colab_type="code" id="LxMVWN-ub42i" outputId="5b5f6456-a706-46b9-e19e-3942d53b01a5"
tup = ('from', 'transform', 'to')
diz = {'SOURCE': 'from', 'LINK': 'transform', 'TARGET': 'to'}

print("TUPLE =================================================")
print(source_getter_tuple(tup))
print(target_getter_tuple(tup))
print(link_getter_tuple(tup))
print(obj_creator_tuple('sss', 'new_link', 'ttt'))
print("DICT ==================================================")
print(source_getter_dict(diz))
print(target_getter_dict(diz))
print(link_getter_dict(diz))
print(obj_creator_dict('sss', 'new_link', 'ttt'))

# + [markdown] colab_type="text" id="QaxJ73nKQiDY"
# ## First trivial example
#
# Let's try a trivial example.
#
# we have the following links:
#
#     T_ORIGINAL_DATA -> V_DATA -> T_DATA_WORK -> T_DATA_FINAL
#
# and we want to end up with:
#     
#     T_ORIGINAL_DATA -> T_DATA_FINAL.
#
# We have to remove the two vertices in the middle.

# + colab={"base_uri": "https://localhost:8080/", "height": 326} colab_type="code" id="eNWPE0mskpwp" outputId="7e981dab-5abf-4e89-fd96-513e758030cf"
# T_ORIGINAL_TABLE -> V_TABLE -> T_TABLE_WORK -> T_TABLE_FINAL
# should become
# T_ORIGINAL_TABLE -> T_TABLE_FINAL

vertices_list = [
    ('T_ORIGINAL_DATA',),
    ('V_DATA',),
    ('T_DATA_WORK',),
    ('T_DATA_FINAL',)
]

links_list = [
    ('T_ORIGINAL_DATA', 'pt', 'V_DATA'),
    ('V_DATA', 'si', 'T_DATA_WORK'),
    ('T_DATA_WORK', 'pt', 'T_DATA_FINAL')
]

vertices_df = spark.createDataFrame(vertices_list, ['ITEM'])
links_df = spark.createDataFrame(links_list, ['SOURCE', 'LINK', 'TARGET'])

vertices_df.show()
links_df.show()

# + [markdown] colab_type="text" id="pZn2zsDfJwUF"
# ### RDD as the collection data model

# + colab={"base_uri": "https://localhost:8080/", "height": 144} colab_type="code" id="cy-A_xDNmqJF" outputId="449f2431-3349-4409-c264-3bdcad1cdfe1"
vertices_rdd = vertices_df.rdd.map(lambda x: x.asDict())
links_rdd = links_df.rdd.map(lambda x: x.asDict())

vertices_to_remove = (vertices_rdd
                      .map(lambda x: x['ITEM'])
                      .filter(lambda x: x.startswith('V') or x.rfind('WORK') != -1)
                      .collect()
                      )
vertices_cleaned_rdd = vertices_rdd.filter(lambda x: x['ITEM'] not in vertices_to_remove)

links_cleaned_rdd = vertex_smoothing(
    links_rdd.repartition(1), vertices_to_remove, source_getter_dict, target_getter_dict,
    link_getter_dict, obj_creator_dict
)
print("RDD -> Links cleaned:", links_cleaned_rdd.collect())

# + colab={"base_uri": "https://localhost:8080/", "height": 1761} colab_type="code" id="wFFxwcc9PJBC" outputId="819b7167-b844-494e-e239-f0fffd714f0f"
print(links_cleaned_rdd.toDebugString().decode('UTF-8'))

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="cqKSzD8vQ9_g" outputId="56c83967-c549-4411-906c-23d251507aa4"
links_cleaned_rdd.getNumPartitions()

# + [markdown] colab_type="text" id="ytK1gS7q2A0U"
# ### LIST as the collection data model

# + colab={} colab_type="code" id="KsQguLvB2ACS"
vertices_to_remove = filter(lambda x: x.startswith('V') or x.rfind('WORK') != -1,
                            map(lambda x: x[0], vertices_list))
vertices_cleaned_list = filter(lambda x: x[0] not in vertices_to_remove, vertices_list)

links_cleaned_list = vertex_smoothing(
    links_list, vertices_to_remove, source_getter_tuple, target_getter_tuple,
    link_getter_tuple, obj_creator_tuple
)
print("LIST -> Links cleaned:", links_cleaned_list)

# + [markdown] colab_type="text" id="SYg-UkvQRjds"
# ## Second (not trivial) example
#
# Let's try a less trivial example.
#
# We have the following links:
#
#      T_ORIGINAL_TABLE_1 -\                            /-> T_TABLE_FINAL_1 
#                           -> V_TABLE -> T_TABLE_WORK -
#      T_ORIGINAL_TABLE_2 -/                            \-> T_TABLE_FINAL_2
#
# and we want to end up with:
#
#      T_ORIGINAL_TABLE_1 -\ /-> T_TABLE_FINAL_1 
#                           X
#      T_ORIGINAL_TABLE_2 -/ \-> T_TABLE_FINAL_2
#
# We have to remove the two vertices in the middle, and build more complex links.

# + colab={"base_uri": "https://localhost:8080/", "height": 399} colab_type="code" id="Iar7UtvNRQ14" outputId="a2771326-c43f-42c5-dd2d-f46e93109336"
# T_ORIGINAL_TABLE_1 -\                            /-> T_TABLE_FINAL_1 
#                      -> V_TABLE -> T_TABLE_WORK -
# T_ORIGINAL_TABLE_2 -/                            \-> T_TABLE_FINAL_2
#
# should become
#
# T_ORIGINAL_TABLE_1 -\ /-> T_TABLE_FINAL_1 
#                      X
# T_ORIGINAL_TABLE_2 -/ \-> T_TABLE_FINAL_2


vertices_list = [
    ("T_ORIGINAL_TABLE_1",),
    ("T_ORIGINAL_TABLE_2",),
    ("V_TABLE",),
    ("T_TABLE_WORK",),
    ("T_TABLE_FINAL_1",),
    ("T_TABLE_FINAL_2",)
]

links_list = [
    ('T_ORIGINAL_TABLE_1', 'mi', 'V_TABLE'),
    ('T_ORIGINAL_TABLE_2', 'mi', 'V_TABLE'),
    ('V_TABLE', 'pt', 'T_TABLE_WORK'),
    ('T_TABLE_WORK', 'si', 'T_TABLE_FINAL_1'),
    ('T_TABLE_WORK', 'si', 'T_TABLE_FINAL_2')
]

vertices_df = spark.createDataFrame(vertices_list, ["ITEM"])
links_df = spark.createDataFrame(links_list, ["SOURCE", "LINK", "TARGET"])

vertices_df.show()
links_df.show()

# + [markdown] colab_type="text" id="3PUDxTvASa7_"
# ### RDD as the collection data model

# + colab={"base_uri": "https://localhost:8080/", "height": 217} colab_type="code" id="Y_7dPyD9SW1h" outputId="568c5af9-a0b3-421c-d5d5-093db735beee"
vertices_rdd = vertices_df.rdd.map(lambda x: x.asDict())
links_rdd = links_df.rdd.map(lambda x: x.asDict())

vertices_to_remove = (vertices_rdd
                      .map(lambda x: x['ITEM'])
                      .filter(lambda x: x.startswith('V') or x.rfind('WORK') != -1)
                      .collect()
                      )
vertices_cleaned_rdd = vertices_rdd.filter(lambda x: x['ITEM'] not in vertices_to_remove)

links_cleaned_rdd = vertex_smoothing(
    links_rdd.repartition(1), vertices_to_remove, source_getter_dict, target_getter_dict,
    link_getter_dict, obj_creator_dict
)
print("RDD -> Links cleaned:", links_cleaned_rdd.collect())

# + [markdown] colab_type="text" id="IxtRQ5T_StMi"
# ### LIST as the collection data model

# + colab={} colab_type="code" id="cn419bfPSovJ"
vertices_to_remove = filter(lambda x: x.startswith('V') or x.rfind('WORK') != -1,
                            map(lambda x: x[0], vertices_list))
vertices_cleaned_list = filter(lambda x: x[0] not in vertices_to_remove, vertices_list)

links_cleaned_list = vertex_smoothing(
    links_list, vertices_to_remove, source_getter_tuple, target_getter_tuple,
    link_getter_tuple, obj_creator_tuple
)
print("LIST -> Links cleaned:", links_cleaned_list)

StarcoderdataJupyterScriptsDedupFiltered