# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="uXIWSN300w5v"
#
# ![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)
#
# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/component_examples/text_pre_processing_and_cleaning/document_normalizer_demo.ipynb)
#
#
#
# The DocumentNormalizer extracts content from HTML or XML documents, applying either data cleansing using an arbitrary number of custom regular expressions either data extraction following the different parameters

# + id="abuB9K1_QVuL" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1619911221148, "user_tz": -120, "elapsed": 124898, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64", "userId": "14469489166467359317"}} outputId="2e87d14b-ff33-491b-91bd-34963a8d5794"
# !wget https://setup.johnsnowlabs.com/nlu/colab.sh -O - | bash
  

import nlu

# + colab={"base_uri": "https://localhost:8080/"} id="4SQzdgwyQ65Q" executionInfo={"status": "ok", "timestamp": 1619911312421, "user_tz": -120, "elapsed": 1485, "user": {"displayName": "<NAME>", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64", "userId": "14469489166467359317"}} outputId="b20f2ef4-3821-42fa-931f-65448afd70cb"
pipe = nlu.load('norm_document')
data = '<!DOCTYPE html> <html> <head> <title>Example</title> </head> <body> <p>This is an example of a simple HTML page with one paragraph.</p> </body> </html>'
df = pipe.predict(data,output_level='document')
print(df['doc_norm'])
print(df.iloc[0]['doc_norm'])

# + id="o73dDzocR7L_"
pipe.print_info()


StarcoderdataJupyterScriptsDedupFiltered