nlpmodels/utils/elt/dataset.py
def padded_string_to_integer(token_list: List[List[str]],
                                 max_sequence_length: int,
                                 vocab: NLPVocabulary) -> List[List[int]]:
        """
        Take a sequence of (string) tokens and convert them to a padded set of integers.

        Args:
            token_list (List[List[str]]):
                List of tokens to be converted to indices.
            max_sequence_length (int):
                Maximum length of sequence for each target, source sequence.
            vocab (NLPVocabulary):
                Dictionary to look up indices for each token.
        Returns:
            Sequence of indicies with EOS and PAD indices.
        """

        integer_list = []

        for tokens in token_list:
            integers = [vocab.mask_index] * max_sequence_length
            # this allows for truncated sequences.
            # In some problems, we will explicitly through out
            # datapoints < max_sequence_length prior to this step.
            integers[:len(tokens)] = [vocab.lookup_token(x) for x in tokens][:len(integers)]
            # Adding in the EOS token if the sequence is not truncated.
            if len(tokens) < max_sequence_length:
                integers[len(tokens)] = vocab.eos_index
            integer_list.append(integers)

        return integer_list

        Take a sequence of (string) tokens and convert them to a padded set of integers.

        Args:
            token_list (List[List[str]]):
                List of tokens to be converted to indices.
            max_sequence_length (int):
                Maximum length of sequence for each target, source sequence.
            vocab (NLPVocabulary):
                Dictionary to look up indices for each token.
        Returns:
            Sequence of indicies with EOS and PAD indices.
        
Take a sequence of (string) tokens and convert them to a padded set of integers.
Take a sequence of ( string ) tokens and convert them to a padded set of integers .
def padded_string_to_integer(token_list: List[List[str]],
                                 max_sequence_length: int,
                                 vocab: NLPVocabulary) -> List[List[int]]:
        integer_list = []
        for tokens in token_list:
            integers = [vocab.mask_index] * max_sequence_length
            integers[:len(tokens)] = [vocab.lookup_token(x) for x in tokens][:len(integers)]
            if len(tokens) < max_sequence_length:
                integers[len(tokens)] = vocab.eos_index
            integer_list.append(integers)
        return integer_list
def padded_string_to_integer ( token_list : List [ List [ str ] ] , max_sequence_length : int , vocab : NLPVocabulary ) -> List [ List [ int ] ] : integer_list = [ ] for tokens in token_list : integers = [ vocab . mask_index ] * max_sequence_length integers [ : len ( tokens ) ] = [ vocab . lookup_token ( x ) for x in tokens ] [ : len ( integers ) ] if len ( tokens ) < max_sequence_length : integers [ len ( tokens ) ] = vocab . eos_index integer_list . append ( integers ) return integer_list
Take a sequence of (string) tokens and convert them to a padded set of integers.
Take a sequence of ( string ) tokens and convert them to a padded set of integers .
"""
        Take a sequence of (string) tokens and convert them to a padded set of integers.

        Args:
            token_list (List[List[str]]):
                List of tokens to be converted to indices.
            max_sequence_length (int):
                Maximum length of sequence for each target, source sequence.
            vocab (NLPVocabulary):
                Dictionary to look up indices for each token.
        Returns:
            Sequence of indicies with EOS and PAD indices.
        """ # this allows for truncated sequences. # In some problems, we will explicitly through out # datapoints < max_sequence_length prior to this step. # Adding in the EOS token if the sequence is not truncated.
Sequence of indicies with EOS and PAD indices.
Sequence of indicies with EOS and PAD indices .
token_list
List[List[str]]
List of tokens to be converted to indices.
List of tokens to be converted to indices .
max_sequence_length
int
Maximum length of sequence for each target, source sequence.
Maximum length of sequence for each target source sequence .
vocab
NLPVocabulary
Dictionary to look up indices for each token.
Dictionary to look up indices for each token .