4.7 - ACE/Analysis/helper_functions.py
def tokenise(text):
    """
    Turns given text into tokens.
    """
    cleaned = clean_text(text)
    cleaned = re.sub(r"(\p{P})\p{P}*", r"\1 ", cleaned)
    tokens = spacy_tokenise(cleaned)
    return tokens

    Turns given text into tokens.
    
Turns given text into tokens.
Turns given text into tokens .
def tokenise(text):
    cleaned = clean_text(text)
    cleaned = re.sub(r"(\p{P})\p{P}*", r"\1 ", cleaned)
    tokens = spacy_tokenise(cleaned)
    return tokens
def tokenise ( text ) : cleaned = clean_text ( text ) cleaned = re . sub ( r"(\p{P})\p{P}*" , r"\1 " , cleaned ) tokens = spacy_tokenise ( cleaned ) return tokens
Turns given text into tokens.
Turns given text into tokens .
"""
    Turns given text into tokens.
    """
text