def _word_ngrams(self, tokens):
<EOL>if self.stop_words is not None:<EOL><INDENT>tokens = [w for w in tokens if w not in self.stop_words]<EOL><DEDENT>min_n, max_n = self.ngram_range<EOL>if max_n != <NUM_LIT:1>:<EOL><INDENT>original_tokens = tokens<EOL>if min_n == <NUM_LIT:1>:<EOL><INDENT>tokens = list(original_tokens)<EOL>min_n += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>tokens = []<EOL><DEDENT>n_original_tokens = len(original_tokens)<EOL>tokens_append = tokens.append<EOL>space_join = "<STR_LIT:U+0020>".join<EOL>for n in range(min_n,<EOL>min(max_n + <NUM_LIT:1>, n_original_tokens + <NUM_LIT:1>)):<EOL><INDENT>for i in range(n_original_tokens - n + <NUM_LIT:1>):<EOL><INDENT>tokens_append(space_join(original_tokens[i: i + n]))<EOL><DEDENT><DEDENT><DEDENT>return tokens<EOL>
Turn tokens into a tokens of n-grams

ref: https://github.com/scikit-learn/scikit-learn/blob/ef5cb84a/sklearn/feature_extraction/text.py#L124-L153