Preprocessing/generate_romanization.py
def from_file(cls, rom_file: str, prop_typical: float = 0.9,
                  encoding: str = 'utf8',
                  # any_possible: bool = False
                  ):
        """
        Creates a Romanizer object from a .txt file containing
        a Romanization key

        Parameters
        ----------
        rom_file : str, file name for romanization key -- 
                see hye_translit_key for formatting
        prop_typical: float, amount of probability weight assigned to more
                probable transliterations; default 0.9

        """
        lines = open(rom_file, encoding=encoding).readlines()
        char_probs = {}
        
        for l in lines:
            if re_comment_line.search(l) == None:
                # strip whitespace
                l = l.strip()
                
                match = re_char_file.search(l)
                
                if match:
                    orig, most_prob_str, less_prob_str = match.groups()
                    
                    # make char list starting with most common
                    chars = [item.strip() for item in 
                            most_prob_str.split(sep=",")]
                    
                    # make prob list (cumulative weight distribution)
                    # if no less prob options, assign all weight to most prob
                    if less_prob_str == '':
                        probs = list(np.linspace(
                            0, 1, num=len(chars)+1)[1:])
                        
                    else:
                        probs = list(np.linspace(
                            0,prop_typical, num=len(chars)+1)[1:])
                        
                        # add less common chars
                        less_common = [item.strip() for item in
                                      less_prob_str.split(sep=",")]
                        chars.extend(less_common)
                        
                        probs.extend(np.linspace(
                            prop_typical, 1,num=len(less_common)+1)[1:])
                        
                    # add character and candidate transliterations to table    
                    char_probs[orig] = tuple([chars, probs])
                    
        return(Romanizer(char_probs, encoding=encoding))

        Creates a Romanizer object from a .txt file containing
        a Romanization key

        Parameters
        ----------
        rom_file : str, file name for romanization key -- 
                see hye_translit_key for formatting
        prop_typical: float, amount of probability weight assigned to more
                probable transliterations; default 0.9

        
Creates a Romanizer object from a .txt file containing
a Romanization key

Parameters

rom_file : str, file name for romanization key
see hye_translit_key for formatting
prop_typical: float, amount of probability weight assigned to more
probable transliterations; default 0.9
Creates a Romanizer object from a . txt file containing a Romanization key Parameters rom_file : str file name for romanization key see hye_translit_key for formatting prop_typical : float amount of probability weight assigned to more probable transliterations ; default 0 . 9
def from_file(cls, rom_file: str, prop_typical: float = 0.9,
                  encoding: str = 'utf8',
                  ):
        lines = open(rom_file, encoding=encoding).readlines()
        char_probs = {}
        for l in lines:
            if re_comment_line.search(l) == None:
                l = l.strip()
                match = re_char_file.search(l)
                if match:
                    orig, most_prob_str, less_prob_str = match.groups()
                    chars = [item.strip() for item in 
                            most_prob_str.split(sep=",")]
                    if less_prob_str == '':
                        probs = list(np.linspace(
                            0, 1, num=len(chars)+1)[1:])
                    else:
                        probs = list(np.linspace(
                            0,prop_typical, num=len(chars)+1)[1:])
                        less_common = [item.strip() for item in
                                      less_prob_str.split(sep=",")]
                        chars.extend(less_common)
                        probs.extend(np.linspace(
                            prop_typical, 1,num=len(less_common)+1)[1:])
                    char_probs[orig] = tuple([chars, probs])
        return(Romanizer(char_probs, encoding=encoding))
def from_file ( cls , rom_file : str , prop_typical : float = 0.9 , encoding : str = 'utf8' , ) : lines = open ( rom_file , encoding = encoding ) . readlines ( ) char_probs = { } for l in lines : if re_comment_line . search ( l ) == None : l = l . strip ( ) match = re_char_file . search ( l ) if match : orig , most_prob_str , less_prob_str = match . groups ( ) chars = [ item . strip ( ) for item in most_prob_str . split ( sep = "," ) ] if less_prob_str == '' : probs = list ( np . linspace ( 0 , 1 , num = len ( chars ) + 1 ) [ 1 : ] ) else : probs = list ( np . linspace ( 0 , prop_typical , num = len ( chars ) + 1 ) [ 1 : ] ) less_common = [ item . strip ( ) for item in less_prob_str . split ( sep = "," ) ] chars . extend ( less_common ) probs . extend ( np . linspace ( prop_typical , 1 , num = len ( less_common ) + 1 ) [ 1 : ] ) char_probs [ orig ] = tuple ( [ chars , probs ] ) return ( Romanizer ( char_probs , encoding = encoding ) )
Creates a Romanizer object from a .txt file containing
a Romanization key
Creates a Romanizer object from a . txt file containing a Romanization key
# any_possible: bool = False """
        Creates a Romanizer object from a .txt file containing
        a Romanization key

        Parameters
        ----------
        rom_file : str, file name for romanization key -- 
                see hye_translit_key for formatting
        prop_typical: float, amount of probability weight assigned to more
                probable transliterations; default 0.9

        """ # strip whitespace # make char list starting with most common # make prob list (cumulative weight distribution) # if no less prob options, assign all weight to most prob # add less common chars # add character and candidate transliterations to table    
cls
rom_file
str
prop_typical
float
encoding
str