# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: markuplmft
#     language: python
#     name: markuplmft
# ---

# # Check for the dataset format

# + tags=[]
import pandas as pd
from pathlib import Path

# + tags=[]
# vertical / website / page
# -

# # Packed Data (Data after pack_data.py)

# ## SWDE data 

# + tags=[]
data_path = '../../../swde/sourceCode/swde_small.pickle'
data_packed = pd.read_pickle(data_path)
len(data_packed)

# + tags=[]
data_packed[:1]

# + [markdown] tags=[]
# ### Ground Truth

# + tags=[]
gt_path = Path('../../../swde/sourceCode/groundtruth/auto/')
gt_file = [x for x in list(gt_path.iterdir())][0] 
with open(gt_file) as text:
    lines = text.readlines()
    for l in lines:
        print(l)
# -

# ## My data

# + tags=[]
data_path = '../../../swde/my_CF_sourceCode/wae.pickle'
data_packed = pd.read_pickle(data_path)
len(data_packed)
# -

# ### Ground Truth

# + tags=[]
gt_path = Path.cwd().parents[2] / 'swde/my_CF_sourceCode/groundtruth/WAE/'

for gt_file in list(gt_path.iterdir())[:]:
    print(gt_file)
    with open(gt_file) as text:
        lines = text.readlines()
        for l in lines:
            print(l)
# -

# # Prepare Data (Data after prepare_data.py)
#

# + [markdown] tags=[]
# ## SWDE data 

# + tags=[]
website_data_path = Path.cwd().parents[2] / 'swde/swde_processed/auto-msn-3.pickle'
df = pd.read_pickle(website_data_path)

# + tags=[]
page_index = '0000'
pd.DataFrame(df[page_index], columns=['text', 'xpath', 'node-type'])

# + [markdown] tags=[]
# ## My data 

# + tags=[]
pd.set_option('max_colwidth', 2000)

website_data_path = Path.cwd().parents[2] / 'swde/my_CF_processed/WAE-blackbaud.com-2000.pickle'

all_data = pd.read_pickle(website_data_path)

# + tags=[]
page_index = '0001'
df = pd.DataFrame(all_data[page_index], columns=['text', 'xpath', 'node-type'])
df
# -





# ---

# +
from 
from lxml import etree

def get_dom_tree(html):
    """Parses a HTML string to a DOMTree.

    We preprocess the html string and use lxml lib to get a tree structure object.

    Args:
      html: the string of the HTML document.
      website: the website name for dealing with special cases.

    Returns:
      A parsed DOMTree object using lxml library.
    """
    cleaner = Cleaner()
    cleaner.javascript = True
    cleaner.style = True
    cleaner.page_structure = False
    html = html.replace("\0", "")  # Delete NULL bytes.
    # Replace the <br> tags with a special token for post-processing the xpaths.
    html = html.replace("<br>", "--BRRB--")
    html = html.replace("<br/>", "--BRRB--")
    html = html.replace("<br />", "--BRRB--")
    html = html.replace("<BR>", "--BRRB--")
    html = html.replace("<BR/>", "--BRRB--")
    html = html.replace("<BR />", "--BRRB--")

    html = clean_format_str(html)
    x = lxml.html.fromstring(html)
    etree_root = cleaner.clean_html(x)
    dom_tree = etree.ElementTree(etree_root)
    return dom_tree

# + tags=[]


etree.ElementTree(etree_root)
x = lxml.html.fromstring(html)

# + tags=[]
websites

# + tags=[]
p = Path('/Users/aimoredutra/GIT/swde/my_CF_sourceCode')
[x for x in p.iterdir()]
# -





StarcoderdataJupyterScriptsDedupFiltered