text_model/data.py
def encode(self, sentence, add_bos_eos=False, max_len=None, split=None):
        """
        Convert a sentence to a list of ids, with special tokens added.

        Args:
            sentence: If `split` is `None`, sentence is a list of tokens.
                Else sentence is a single string with tokens separated by `split`
            add_bos_eos: If `True`, BOS/EOS will be added.
            max_len: If not `None`, the return token ids exceed `max_len` will be truncated.
        Returns:
            A numpy array of token ids for the input sentence.
        """
        if split:
            sentence = sentence.split(split)
        if add_bos_eos:
            sentence = [Vocabulary.BOS] + sentence + [Vocabulary.EOS]
        word_ids = [ self.word_to_id(word) for word in sentence ]

        return np.array(word_ids[:max_len], dtype=NP_DTYPE_INT)

        Convert a sentence to a list of ids, with special tokens added.

        Args:
            sentence: If `split` is `None`, sentence is a list of tokens.
                Else sentence is a single string with tokens separated by `split`
            add_bos_eos: If `True`, BOS/EOS will be added.
            max_len: If not `None`, the return token ids exceed `max_len` will be truncated.
        Returns:
            A numpy array of token ids for the input sentence.
        
Convert a sentence to a list of ids, with special tokens added.
Convert a sentence to a list of ids with special tokens added .
def encode(self, sentence, add_bos_eos=False, max_len=None, split=None):
        if split:
            sentence = sentence.split(split)
        if add_bos_eos:
            sentence = [Vocabulary.BOS] + sentence + [Vocabulary.EOS]
        word_ids = [ self.word_to_id(word) for word in sentence ]
        return np.array(word_ids[:max_len], dtype=NP_DTYPE_INT)
def encode ( self , sentence , add_bos_eos = False , max_len = None , split = None ) : if split : sentence = sentence . split ( split ) if add_bos_eos : sentence = [ Vocabulary . BOS ] + sentence + [ Vocabulary . EOS ] word_ids = [ self . word_to_id ( word ) for word in sentence ] return np . array ( word_ids [ : max_len ] , dtype = NP_DTYPE_INT )
Convert a sentence to a list of ids, with special tokens added.
Convert a sentence to a list of ids with special tokens added .
"""
        Convert a sentence to a list of ids, with special tokens added.

        Args:
            sentence: If `split` is `None`, sentence is a list of tokens.
                Else sentence is a single string with tokens separated by `split`
            add_bos_eos: If `True`, BOS/EOS will be added.
            max_len: If not `None`, the return token ids exceed `max_len` will be truncated.
        Returns:
            A numpy array of token ids for the input sentence.
        """
A numpy array of token ids for the input sentence.
A numpy array of token ids for the input sentence .
self
sentence
If `split` is `None`, sentence is a list of tokens.
Else sentence is a single string with tokens separated by `split`
If ` split ` is ` None ` sentence is a list of tokens . Else sentence is a single string with tokens separated by ` split `
add_bos_eos
If `True`, BOS/EOS will be added.
If ` True ` BOS / EOS will be added .
max_len
If not `None`, the return token ids exceed `max_len` will be truncated.
If not ` None ` the return token ids exceed ` max_len ` will be truncated .
split