utils.py
def encode(self, text):
        """Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.LongTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.LongTensor [n]: length of each text.
        """

        length = []
        result = []
        for item in text:            
            item = item.decode('utf-8','strict')
            length.append(len(item))
            r = []
            for char in item:
                index = self.dict[char]
                # result.append(index)
                r.append(index)
            result.append(r)
        
        max_len = 0
        for r in result:
            if len(r) > max_len:
                max_len = len(r)
        
        result_temp = []
        for r in result:
            for i in range(max_len - len(r)):
                r.append(0)
            result_temp.append(r)

        text = result_temp
        return (torch.LongTensor(text), torch.LongTensor(length))
Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.LongTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.LongTensor [n]: length of each text.
        
Support batch or single str.
Support batch or single str .
def encode(self, text):
        length = []
        result = []
        for item in text:            
            item = item.decode('utf-8','strict')
            length.append(len(item))
            r = []
            for char in item:
                index = self.dict[char]
                r.append(index)
            result.append(r)
        max_len = 0
        for r in result:
            if len(r) > max_len:
                max_len = len(r)
        result_temp = []
        for r in result:
            for i in range(max_len - len(r)):
                r.append(0)
            result_temp.append(r)
        text = result_temp
        return (torch.LongTensor(text), torch.LongTensor(length))
def encode ( self , text ) : length = [ ] result = [ ] for item in text : item = item . decode ( 'utf-8' , 'strict' ) length . append ( len ( item ) ) r = [ ] for char in item : index = self . dict [ char ] r . append ( index ) result . append ( r ) max_len = 0 for r in result : if len ( r ) > max_len : max_len = len ( r ) result_temp = [ ] for r in result : for i in range ( max_len - len ( r ) ) : r . append ( 0 ) result_temp . append ( r ) text = result_temp return ( torch . LongTensor ( text ) , torch . LongTensor ( length ) )
Support batch or single str.
Support batch or single str .
"""Support batch or single str.

        Args:
            text (str or list of str): texts to convert.

        Returns:
            torch.LongTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
            torch.LongTensor [n]: length of each text.
        """ # result.append(index)
encoded texts.
torch.LongTensor [n]: length of each text.
encoded texts . torch . LongTensor [ n ] : length of each text .
torch.LongTensor [length_0 + length_1 + ... length_{n - 1}]
self
text
texts to convert.
texts to convert .