def METHOD_NAME(self, text: str) -> List[int]:
    tokens = nltk.word_tokenize(text, preserve_line=True)
    #phrase detection
    for ngram_len in sorted(self.ngram_lengths, reverse=True):
        idx = 0
        while idx <= len(tokens) - ngram_len:
            ngram = self.ngram_separator.join(tokens[idx:idx + ngram_len])
            if ngram in self.ngram_lookup:
                tokens[idx:idx + ngram_len] = [ngram]
            elif ngram.lower() in self.ngram_lookup:
                tokens[idx:idx + ngram_len] = [ngram.lower()]
            idx += 1
    #Map tokens to idx, filter stop words
    tokens_filtered = []
    for token in tokens:
        if token in self.stop_words:
            continue
        elif token in self.word2idx:
            tokens_filtered.append(self.word2idx[token])
            continue
        token = token.lower()
        if token in self.stop_words:
            continue
        elif token in self.word2idx:
            tokens_filtered.append(self.word2idx[token])
            continue
        token = token.strip(string.punctuation)
        if token in self.stop_words:
            continue
        elif len(token) > 0 and token in self.word2idx:
            tokens_filtered.append(self.word2idx[token])
            continue
    return tokens_filtered