montreal_forced_aligner/language_modeling/trainer.py
def normalized_text_iter(self, min_count: int = 1) -> Generator:
        """
        Construct an iterator over the normalized texts in the corpus

        Parameters
        ----------
        min_count: int
            Minimum word count to include in the output, otherwise will use OOV code, defaults to 1

        Yields
        -------
        str
            Normalized text
        """
        unk_words = {k for k, v in self.word_counts.items() if v <= min_count}
        for u in self.utterances:
            text = u.text.split()
            new_text = []
            for t in text:
                if u.speaker.dictionary is not None:
                    u.speaker.dictionary.to_int(t)
                    lookup = u.speaker.dictionary.split_clitics(t)
                    if lookup is None:
                        continue
                else:
                    lookup = [t]
                for item in lookup:
                    if item in unk_words:
                        new_text.append(self.oov_word)
                        self.oovs_found[item] += 1
                    elif (
                        u.speaker.dictionary is not None and item not in u.speaker.dictionary.words
                    ):
                        new_text.append(self.oov_word)
                    else:
                        new_text.append(item)
            yield " ".join(new_text)

        Construct an iterator over the normalized texts in the corpus

        Parameters
        ----------
        min_count: int
            Minimum word count to include in the output, otherwise will use OOV code, defaults to 1

        Yields
        -------
        str
            Normalized text
        
Construct an iterator over the normalized texts in the corpus
Parameters

int
Minimum word count to include in the output, otherwise will use OOV code, defaults to 1

Yields

str
Normalized text
Construct an iterator over the normalized texts in the corpus Parameters int Minimum word count to include in the output otherwise will use OOV code defaults to 1 Yields str Normalized text
def normalized_text_iter(self, min_count: int = 1) -> Generator:
        unk_words = {k for k, v in self.word_counts.items() if v <= min_count}
        for u in self.utterances:
            text = u.text.split()
            new_text = []
            for t in text:
                if u.speaker.dictionary is not None:
                    u.speaker.dictionary.to_int(t)
                    lookup = u.speaker.dictionary.split_clitics(t)
                    if lookup is None:
                        continue
                else:
                    lookup = [t]
                for item in lookup:
                    if item in unk_words:
                        new_text.append(self.oov_word)
                        self.oovs_found[item] += 1
                    elif (
                        u.speaker.dictionary is not None and item not in u.speaker.dictionary.words
                    ):
                        new_text.append(self.oov_word)
                    else:
                        new_text.append(item)
            yield " ".join(new_text)
def normalized_text_iter ( self , min_count : int = 1 ) -> Generator : unk_words = { k for k , v in self . word_counts . items ( ) if v <= min_count } for u in self . utterances : text = u . text . split ( ) new_text = [ ] for t in text : if u . speaker . dictionary is not None : u . speaker . dictionary . to_int ( t ) lookup = u . speaker . dictionary . split_clitics ( t ) if lookup is None : continue else : lookup = [ t ] for item in lookup : if item in unk_words : new_text . append ( self . oov_word ) self . oovs_found [ item ] += 1 elif ( u . speaker . dictionary is not None and item not in u . speaker . dictionary . words ) : new_text . append ( self . oov_word ) else : new_text . append ( item ) yield " " . join ( new_text )
Construct an iterator over the normalized texts in the corpus
Parameters
Construct an iterator over the normalized texts in the corpus Parameters
"""
        Construct an iterator over the normalized texts in the corpus

        Parameters
        ----------
        min_count: int
            Minimum word count to include in the output, otherwise will use OOV code, defaults to 1

        Yields
        -------
        str
            Normalized text
        """
self
min_count
int