zeeguu/core/language/strategies/cognacy_difficulty_estimator.py
def estimate_difficulty(self, text: str):
        """
        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        """

        # split and stem words
        words = split_words_from_text(text)
        stemmer = SnowballStemmer(self.language.name.lower())
        words = [stemmer.stem(w.lower()) for w in words]

        # frequency and length
        word_frequency = Counter(words)
        total_words = len(words)

        # score per word
        word_scores = {w: self.word_difficulty(self.score_map, True, w) for w in word_frequency}

        if (len(word_scores)) == 0:

            difficulty_scores = dict(
                median=1.0,
                median_unique=1.0,
                normalized=1.0,
                discrete= "HARD",
                unique_ratio=1.0
            )
        else:

            # words above median score
            word_scores_sorted= sorted(word_scores.items(), key=lambda item: item[1])
            center = int(round(len(word_scores_sorted) / 2, 0))
            word_scores_median = dict(word_scores_sorted[center:])
            total_words_median = sum([word_frequency[w] for w in word_scores_median])

            difficulty_scores = dict(
                median=sum([ s*word_frequency[w] for w, s in word_scores_median.items()])/total_words_median,
                median_unique=mean(list(word_scores_median.values())),
                normalized=sum([ s*word_frequency[w] for w, s in word_scores.items()])/total_words,             # Originally called 'score_average'
                #todo: discrete=discrete_difficulty,               # Originally called 'estimated_difficulty'
                unique_ratio=mean(list(word_scores.values()))      # Ratio of non-cognates
            )

        return difficulty_scores

        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        
This estimator computes the difficulty based on the scoring map
This estimator computes the difficulty based on the scoring map
def estimate_difficulty(self, text: str):
        words = split_words_from_text(text)
        stemmer = SnowballStemmer(self.language.name.lower())
        words = [stemmer.stem(w.lower()) for w in words]
        word_frequency = Counter(words)
        total_words = len(words)
        word_scores = {w: self.word_difficulty(self.score_map, True, w) for w in word_frequency}
        if (len(word_scores)) == 0:
            difficulty_scores = dict(
                median=1.0,
                median_unique=1.0,
                normalized=1.0,
                discrete= "HARD",
                unique_ratio=1.0
            )
        else:
            word_scores_sorted= sorted(word_scores.items(), key=lambda item: item[1])
            center = int(round(len(word_scores_sorted) / 2, 0))
            word_scores_median = dict(word_scores_sorted[center:])
            total_words_median = sum([word_frequency[w] for w in word_scores_median])
            difficulty_scores = dict(
                median=sum([ s*word_frequency[w] for w, s in word_scores_median.items()])/total_words_median,
                median_unique=mean(list(word_scores_median.values())),
                normalized=sum([ s*word_frequency[w] for w, s in word_scores.items()])/total_words,             
                unique_ratio=mean(list(word_scores.values()))      
            )
        return difficulty_scores
def estimate_difficulty ( self , text : str ) : words = split_words_from_text ( text ) stemmer = SnowballStemmer ( self . language . name . lower ( ) ) words = [ stemmer . stem ( w . lower ( ) ) for w in words ] word_frequency = Counter ( words ) total_words = len ( words ) word_scores = { w : self . word_difficulty ( self . score_map , True , w ) for w in word_frequency } if ( len ( word_scores ) ) == 0 : difficulty_scores = dict ( median = 1.0 , median_unique = 1.0 , normalized = 1.0 , discrete = "HARD" , unique_ratio = 1.0 ) else : word_scores_sorted = sorted ( word_scores . items ( ) , key = lambda item : item [ 1 ] ) center = int ( round ( len ( word_scores_sorted ) / 2 , 0 ) ) word_scores_median = dict ( word_scores_sorted [ center : ] ) total_words_median = sum ( [ word_frequency [ w ] for w in word_scores_median ] ) difficulty_scores = dict ( median = sum ( [ s * word_frequency [ w ] for w , s in word_scores_median . items ( ) ] ) / total_words_median , median_unique = mean ( list ( word_scores_median . values ( ) ) ) , normalized = sum ( [ s * word_frequency [ w ] for w , s in word_scores . items ( ) ] ) / total_words , unique_ratio = mean ( list ( word_scores . values ( ) ) ) ) return difficulty_scores
This estimator computes the difficulty based on the scoring map
This estimator computes the difficulty based on the scoring map
"""
        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        """ # split and stem words # frequency and length # score per word # words above median score # Originally called 'score_average' #todo: discrete=discrete_difficulty,               # Originally called 'estimated_difficulty' # Ratio of non-cognates
The dictionary contains the keys and return types
normalized: float (0<=normalized<=1)
discrete: string [EASY, MEDIUM, HARD]
The dictionary contains the keys and return types normalized : float ( 0< = normalized< = 1 ) discrete : string [ EASY MEDIUM HARD ]
dict
self
text
str
None