guise/nlp.py
def stem_based_word_mapping(
    words: Words, stemmer: StemmerSpec = 'lancaster'
) -> WordMap:
    """Returns a dict of {word: replace_with_word, ...} mapping based on stemming.

    stemming is great to reduce the number of words by replacing several


    The words (or word counts) are stemmed, then for each stem, the word with the
    highest count gets to represent the others.

    >>> _words = ['happy', 'happier', 'happiest', 'happily', 'sad', 'sadly']
    >>> word_count = Counter(_words)
    >>> stem_based_word_mapping(word_count)  # doctest: +NORMALIZE_WHITESPACE
    {'happy': 'happy',
     'happier': 'happy',
     'happiest': 'happiest',
     'happily': 'happy',
     'sad': 'sad',
     'sadly': 'sad'}
    """
    if not callable(stemmer):
        from nltk.stem import PorterStemmer
        from nltk.stem import LancasterStemmer

        if stemmer is None:
            stemmer = lambda word: word
        elif isinstance(stemmer, str):
            if stemmer.lower().startswith('porter'):
                stemmer = PorterStemmer().stem
            elif stemmer.lower().startswith('lancaster'):
                stemmer = LancasterStemmer().stem
            else:
                raise ValueError(f'Unknown stemmer value: {stemmer}')
        else:
            raise TypeError(f'Unknown stemmer type: {stemmer}')

    if isinstance(words, Mapping):
        word_count = words
    else:
        word_count = Counter(words)

    word_of_stem = defaultdict(list)
    for word, count in word_count.items():
        word_of_stem[stemmer(word)].append((count, word))

    def sort_key(count_and_word):
        count, word = count_and_word
        # favor the highest count, if not, the shortest word, if not lexicographic
        return -count, len(word), word

    word_of_stem = {
        stem: sorted(count_and_words, key=sort_key)[0][1]
        for stem, count_and_words in word_of_stem.items()
    }

    return {word: word_of_stem[stemmer(word)] for word in word_count}
Returns a dict of {word: replace_with_word, ...} mapping based on stemming.

    stemming is great to reduce the number of words by replacing several


    The words (or word counts) are stemmed, then for each stem, the word with the
    highest count gets to represent the others.

    >>> _words = ['happy', 'happier', 'happiest', 'happily', 'sad', 'sadly']
    >>> word_count = Counter(_words)
    >>> stem_based_word_mapping(word_count)  # doctest: +NORMALIZE_WHITESPACE
    {'happy': 'happy',
     'happier': 'happy',
     'happiest': 'happiest',
     'happily': 'happy',
     'sad': 'sad',
     'sadly': 'sad'}
    
Returns a dict of {word: replace_with_word, ...} mapping based on stemming.
stemming is great to reduce the number of words by replacing several

The words (or word counts) are stemmed, then for each stem, the word with the
highest count gets to represent the others.


Returns a dict of { word : replace_with_word ... } mapping based on stemming . stemming is great to reduce the number of words by replacing several The words ( or word counts ) are stemmed then for each stem the word with the highest count gets to represent the others .
def stem_based_word_mapping(
    words: Words, stemmer: StemmerSpec = 'lancaster'
) -> WordMap:
    if not callable(stemmer):
        from nltk.stem import PorterStemmer
        from nltk.stem import LancasterStemmer
        if stemmer is None:
            stemmer = lambda word: word
        elif isinstance(stemmer, str):
            if stemmer.lower().startswith('porter'):
                stemmer = PorterStemmer().stem
            elif stemmer.lower().startswith('lancaster'):
                stemmer = LancasterStemmer().stem
            else:
                raise ValueError(f'Unknown stemmer value: {stemmer}')
        else:
            raise TypeError(f'Unknown stemmer type: {stemmer}')
    if isinstance(words, Mapping):
        word_count = words
    else:
        word_count = Counter(words)
    word_of_stem = defaultdict(list)
    for word, count in word_count.items():
        word_of_stem[stemmer(word)].append((count, word))
    def sort_key(count_and_word):
        count, word = count_and_word
        return -count, len(word), word
    word_of_stem = {
        stem: sorted(count_and_words, key=sort_key)[0][1]
        for stem, count_and_words in word_of_stem.items()
    }
    return {word: word_of_stem[stemmer(word)] for word in word_count}
def stem_based_word_mapping ( words : Words , stemmer : StemmerSpec = 'lancaster' ) -> WordMap : if not callable ( stemmer ) : from nltk . stem import PorterStemmer from nltk . stem import LancasterStemmer if stemmer is None : stemmer = lambda word : word elif isinstance ( stemmer , str ) : if stemmer . lower ( ) . startswith ( 'porter' ) : stemmer = PorterStemmer ( ) . stem elif stemmer . lower ( ) . startswith ( 'lancaster' ) : stemmer = LancasterStemmer ( ) . stem else : raise ValueError ( f'Unknown stemmer value: {stemmer}' ) else : raise TypeError ( f'Unknown stemmer type: {stemmer}' ) if isinstance ( words , Mapping ) : word_count = words else : word_count = Counter ( words ) word_of_stem = defaultdict ( list ) for word , count in word_count . items ( ) : word_of_stem [ stemmer ( word ) ] . append ( ( count , word ) ) def sort_key ( count_and_word ) : count , word = count_and_word return - count , len ( word ) , word word_of_stem = { stem : sorted ( count_and_words , key = sort_key ) [ 0 ] [ 1 ] for stem , count_and_words in word_of_stem . items ( ) } return { word : word_of_stem [ stemmer ( word ) ] for word in word_count }
Returns a dict of {word: replace_with_word, ...} mapping based on stemming.
Returns a dict of { word : replace_with_word ... } mapping based on stemming .
"""Returns a dict of {word: replace_with_word, ...} mapping based on stemming.

    stemming is great to reduce the number of words by replacing several


    The words (or word counts) are stemmed, then for each stem, the word with the
    highest count gets to represent the others.

    >>> _words = ['happy', 'happier', 'happiest', 'happily', 'sad', 'sadly']
    >>> word_count = Counter(_words)
    >>> stem_based_word_mapping(word_count)  # doctest: +NORMALIZE_WHITESPACE
    {'happy': 'happy',
     'happier': 'happy',
     'happiest': 'happiest',
     'happily': 'happy',
     'sad': 'sad',
     'sadly': 'sad'}
    """ # favor the highest count, if not, the shortest word, if not lexicographic
words
Words
stemmer
StemmerSpec