# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import nltk
tokenizer = nltk.tokenize.TreebankWordTokenizer()

text = """
... that oak trees associated with <NAME> stood in Elderslie (pictured), 
Port Glasgow and Torwood in Scotland?
"""

tokens = tokenizer.tokenize(text)
tokens

stemmer = nltk.stem.PorterStemmer()
stemmed_text = " ".join(stemmer.stem(token) for token in tokens)
stemmed_text

stemmer = nltk.stem.WordNetLemmatizer()
stemmed_text = " ".join(stemmer.lemmatize(token) for token in tokens)
stemmed_text



StarcoderdataJupyterScriptsDedupFiltered