Model/vocab.py
def from_files(cls, files: list[str], 
                         enc = "utf8", **kwargs):
        """
        Builds a character-level vocabulary from a list of files.
        Returns a Vocabulary object

        """
        
        tokens = Counter()
        
        for f in files:
            if Path(f).is_file():
                with open(f, "r", encoding=enc) as file:
                    lines = file.readlines()
                    lines = [''.join(line.split()) for line in lines]
                    
                    for line in lines:
                        chars = list(line)
                        tokens.update(chars)
                        
        return(cls(tokens, **kwargs))

        Builds a character-level vocabulary from a list of files.
        Returns a Vocabulary object

        
Builds a character-level vocabulary from a list of files.
Returns a Vocabulary object
Builds a character - level vocabulary from a list of files . Returns a Vocabulary object
def from_files(cls, files: list[str], 
                         enc = "utf8", **kwargs):
        tokens = Counter()
        for f in files:
            if Path(f).is_file():
                with open(f, "r", encoding=enc) as file:
                    lines = file.readlines()
                    lines = [''.join(line.split()) for line in lines]
                    for line in lines:
                        chars = list(line)
                        tokens.update(chars)
        return(cls(tokens, **kwargs))
def from_files ( cls , files : list [ str ] , enc = "utf8" , ** kwargs ) : tokens = Counter ( ) for f in files : if Path ( f ) . is_file ( ) : with open ( f , "r" , encoding = enc ) as file : lines = file . readlines ( ) lines = [ '' . join ( line . split ( ) ) for line in lines ] for line in lines : chars = list ( line ) tokens . update ( chars ) return ( cls ( tokens , ** kwargs ) )
Builds a character-level vocabulary from a list of files.
Builds a character - level vocabulary from a list of files .
"""
        Builds a character-level vocabulary from a list of files.
        Returns a Vocabulary object

        """
cls
files
list[str]
enc