def tokenize(sentence):
tokens = []<EOL>class Vars:<EOL><INDENT>start_pos = -<NUM_LIT:1><EOL>last_type = '<STR_LIT:o>'<EOL><DEDENT>def update(c, i):<EOL><INDENT>if c.isalpha() or c in '<STR_LIT>':<EOL><INDENT>t = '<STR_LIT:a>'<EOL><DEDENT>elif c.isdigit() or c == '<STR_LIT:#>':<EOL><INDENT>t = '<STR_LIT:n>'<EOL><DEDENT>elif c.isspace():<EOL><INDENT>t = '<STR_LIT:s>'<EOL><DEDENT>else:<EOL><INDENT>t = '<STR_LIT:o>'<EOL><DEDENT>if t != Vars.last_type or t == '<STR_LIT:o>':<EOL><INDENT>if Vars.start_pos >= <NUM_LIT:0>:<EOL><INDENT>token = sentence[Vars.start_pos:i].lower()<EOL>if token not in '<STR_LIT>':<EOL><INDENT>tokens.append(token)<EOL><DEDENT><DEDENT>Vars.start_pos = -<NUM_LIT:1> if t == '<STR_LIT:s>' else i<EOL><DEDENT>Vars.last_type = t<EOL><DEDENT>for i, char in enumerate(sentence):<EOL><INDENT>update(char, i)<EOL><DEDENT>update('<STR_LIT:U+0020>', len(sentence))<EOL>return tokens<EOL>
Converts a single sentence into a list of individual significant units
Args:
    sentence (str): Input string ie. 'This is a sentence.'
Returns:
    list<str>: List of tokens ie. ['this', 'is', 'a', 'sentence']