# stopwords = pkgutil.get_data(__package__, 'smart_common_words.txt.txt')
# stopwords = stopwords.decode('ascii').split('\n')
# stopwords = {key.strip(): 1 for key in stopwords}
import nltk
from nltk.corpus import stopwords

stop = set(stopwords.words('english'))


def _get_ngrams(n, text):
    """Calcualtes n-grams.

    Args:
      n: which n-grams to calculate
      text: An array of tokens

    Returns:
      A set of n-grams
    """
    ngram_set = set()
    text_length = len(text)
    max_index_ngram_start = text_length - n
    for i in range(max_index_ngram_start + 1):
        ngram_set.add(tuple(text[i:i + n]))
    return ngram_set


def _get_word_ngrams(n, sentences, rm_stop_unigram=True):
    """Calculates word n-grams for multiple sentences.
    """
    assert len(sentences) > 0
    assert n > 0

    # words = _split_into_words(sentences)

    words = sum(sentences, [])
    if rm_stop_unigram and n == 1:
        words = set(words)
        words = list(words.difference(stop))
        # words = [w for w in words if w not in stop]
    return _get_ngrams(n, words)
