zeeguu/core/language/strategies/word_history_difficulty_estimator.py
def estimate_difficulty(self, text: str):
        """
        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        """

        # Calculate difficulty for each word
        words = split_words_from_text(text)
        stemmer = SnowballStemmer(self.language.name.lower())
        words = [stemmer.stem(w.lower()) for w in words]

        # frequency and length
        word_frequency = Counter(words)
        total_words = len(words)


        # score per word
        word_scores = {w: self.word_difficulty(self.score_map, True, w) for w in word_frequency}


        # If we can't compute the text difficulty, we estimate hard
        if (len(word_scores)) == 0:
            difficulty_scores = dict(
                median=1.0,
                median_unique=1.0,
                normalized=1.0,
                discrete= "HARD",
                unique_ratio=1.0
            )
        else:

            # words above median score
            word_scores_sorted= sorted(word_scores.items(), key=lambda item: item[1])
            center = int(round(len(word_scores_sorted) / 2, 0))
            word_scores_median = dict(word_scores_sorted[center:])
            total_words_median = sum([word_frequency[w] for w in word_scores_median])

            # If we can't compute the text difficulty, we estimate hard

            difficulty_scores = dict(
                median=sum([ s*word_frequency[w] for w, s in word_scores_median.items()])/total_words_median,
                median_unique=mean(list(word_scores_median.values())),
                normalized=sum([ s*word_frequency[w] for w, s in word_scores.items()])/total_words,  # Originally called 'score_average'
                #todo: discrete=discrete_difficulty,               # Originally called 'estimated_difficulty'
                unique_ratio=mean(list(word_scores.values()))      # Ratio of unknown
            )

        return difficulty_scores

        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        
This estimator computes the difficulty based on the scoring map
This estimator computes the difficulty based on the scoring map
def estimate_difficulty(self, text: str):
        words = split_words_from_text(text)
        stemmer = SnowballStemmer(self.language.name.lower())
        words = [stemmer.stem(w.lower()) for w in words]
        word_frequency = Counter(words)
        total_words = len(words)
        word_scores = {w: self.word_difficulty(self.score_map, True, w) for w in word_frequency}
        if (len(word_scores)) == 0:
            difficulty_scores = dict(
                median=1.0,
                median_unique=1.0,
                normalized=1.0,
                discrete= "HARD",
                unique_ratio=1.0
            )
        else:
            word_scores_sorted= sorted(word_scores.items(), key=lambda item: item[1])
            center = int(round(len(word_scores_sorted) / 2, 0))
            word_scores_median = dict(word_scores_sorted[center:])
            total_words_median = sum([word_frequency[w] for w in word_scores_median])
            difficulty_scores = dict(
                median=sum([ s*word_frequency[w] for w, s in word_scores_median.items()])/total_words_median,
                median_unique=mean(list(word_scores_median.values())),
                normalized=sum([ s*word_frequency[w] for w, s in word_scores.items()])/total_words,  
                unique_ratio=mean(list(word_scores.values()))      
            )
        return difficulty_scores
def estimate_difficulty ( self , text : str ) : words = split_words_from_text ( text ) stemmer = SnowballStemmer ( self . language . name . lower ( ) ) words = [ stemmer . stem ( w . lower ( ) ) for w in words ] word_frequency = Counter ( words ) total_words = len ( words ) word_scores = { w : self . word_difficulty ( self . score_map , True , w ) for w in word_frequency } if ( len ( word_scores ) ) == 0 : difficulty_scores = dict ( median = 1.0 , median_unique = 1.0 , normalized = 1.0 , discrete = "HARD" , unique_ratio = 1.0 ) else : word_scores_sorted = sorted ( word_scores . items ( ) , key = lambda item : item [ 1 ] ) center = int ( round ( len ( word_scores_sorted ) / 2 , 0 ) ) word_scores_median = dict ( word_scores_sorted [ center : ] ) total_words_median = sum ( [ word_frequency [ w ] for w in word_scores_median ] ) difficulty_scores = dict ( median = sum ( [ s * word_frequency [ w ] for w , s in word_scores_median . items ( ) ] ) / total_words_median , median_unique = mean ( list ( word_scores_median . values ( ) ) ) , normalized = sum ( [ s * word_frequency [ w ] for w , s in word_scores . items ( ) ] ) / total_words , unique_ratio = mean ( list ( word_scores . values ( ) ) ) ) return difficulty_scores
This estimator computes the difficulty based on the scoring map
This estimator computes the difficulty based on the scoring map
"""
        This estimator computes the difficulty based on the scoring map
        :param text: See DifficultyEstimatorStrategy
        :rtype: dict
        :return: The dictionary contains the keys and return types
                    normalized: float (0<=normalized<=1)
                    discrete: string [EASY, MEDIUM, HARD]
        """ # Calculate difficulty for each word # frequency and length # score per word # If we can't compute the text difficulty, we estimate hard # words above median score # If we can't compute the text difficulty, we estimate hard # Originally called 'score_average' #todo: discrete=discrete_difficulty,               # Originally called 'estimated_difficulty' # Ratio of unknown
The dictionary contains the keys and return types
normalized: float (0<=normalized<=1)
discrete: string [EASY, MEDIUM, HARD]
The dictionary contains the keys and return types normalized : float ( 0< = normalized< = 1 ) discrete : string [ EASY MEDIUM HARD ]
dict
self
text
str
None