sentence_transformers/datasets/SentenceLabelDataset.py
def convert_input_examples(self, examples: List[InputExample], model: SentenceTransformer):
        """
        Converts input examples to a SentenceLabelDataset.

        Assumes only one sentence per InputExample and labels as integers from 0 to max_num_labels
        and should be used in combination with dataset_reader.LabelSentenceReader.

        Labels with only one example are ignored.

        :param examples:
            the input examples for the training
        :param model
            the Sentence Transformer model for the conversion
        :param is_pretokenized
            If set to true, no tokenization will be applied. It is expected that the input is tokenized via model.tokenize
        """

        inputs = []
        labels = []

        label_sent_mapping = {}
        too_long = 0
        label_type = None

        logging.info("Start tokenization")
        if not self.parallel_tokenization or self.max_processes == 1 or len(examples) <= self.chunk_size:
            tokenized_texts = [self.tokenize_example(example) for example in examples]
        else:
            logging.info("Use multi-process tokenization with {} processes".format(self.max_processes))
            self.model.to('cpu')
            with Pool(self.max_processes) as p:
                tokenized_texts = list(p.imap(self.tokenize_example, examples, chunksize=self.chunk_size))

        # Group examples and labels
        # Add examples with the same label to the same dict
        for ex_index, example in enumerate(tqdm(examples, desc="Convert dataset")):
            if label_type is None:
                if isinstance(example.label, int):
                    label_type = torch.long
                elif isinstance(example.label, float):
                    label_type = torch.float
            tokenized_text = tokenized_texts[ex_index][0]

            if hasattr(model, 'max_seq_length') and model.max_seq_length is not None and model.max_seq_length > 0 and len(tokenized_text) > model.max_seq_length:
                too_long += 1

            if example.label in label_sent_mapping:
                label_sent_mapping[example.label].append(ex_index)
            else:
                label_sent_mapping[example.label] = [ex_index]

            inputs.append(tokenized_text)
            labels.append(example.label)

        # Group sentences, such that sentences with the same label
        # are besides each other. Only take labels with at least 2 examples
        distinct_labels = list(label_sent_mapping.keys())
        for i in range(len(distinct_labels)):
            label = distinct_labels[i]
            if len(label_sent_mapping[label]) >= 2:
                self.grouped_inputs.extend([inputs[j] for j in label_sent_mapping[label]])
                self.grouped_labels.extend([labels[j] for j in label_sent_mapping[label]])
                self.groups_right_border.append(len(self.grouped_inputs)) #At which position does this label group / bucket end?
                self.num_labels += 1

        self.grouped_labels = torch.tensor(self.grouped_labels, dtype=label_type)
        logging.info("Num sentences: %d" % (len(self.grouped_inputs)))
        logging.info("Sentences longer than max_seqence_length: {}".format(too_long))
        logging.info("Number of labels with >1 examples: {}".format(len(distinct_labels)))

        Converts input examples to a SentenceLabelDataset.

        Assumes only one sentence per InputExample and labels as integers from 0 to max_num_labels
        and should be used in combination with dataset_reader.LabelSentenceReader.

        Labels with only one example are ignored.

        :param examples:
            the input examples for the training
        :param model
            the Sentence Transformer model for the conversion
        :param is_pretokenized
            If set to true, no tokenization will be applied. It is expected that the input is tokenized via model.tokenize
        
Converts input examples to a SentenceLabelDataset.
Assumes only one sentence per InputExample and labels as integers from 0 to max_num_labels
and should be used in combination with dataset_reader.LabelSentenceReader.

Labels with only one example are ignored.

:param examples:
the input examples for the training
:param model
the Sentence Transformer model for the conversion
:param is_pretokenized
If set to true, no tokenization will be applied. It is expected that the input is tokenized via model.tokenize
Converts input examples to a SentenceLabelDataset . Assumes only one sentence per InputExample and labels as integers from 0 to max_num_labels and should be used in combination with dataset_reader . LabelSentenceReader . Labels with only one example are ignored . : param examples : the input examples for the training : param model the Sentence Transformer model for the conversion : param is_pretokenized If set to true no tokenization will be applied . It is expected that the input is tokenized via model . tokenize
def convert_input_examples(self, examples: List[InputExample], model: SentenceTransformer):
        inputs = []
        labels = []
        label_sent_mapping = {}
        too_long = 0
        label_type = None
        logging.info("Start tokenization")
        if not self.parallel_tokenization or self.max_processes == 1 or len(examples) <= self.chunk_size:
            tokenized_texts = [self.tokenize_example(example) for example in examples]
        else:
            logging.info("Use multi-process tokenization with {} processes".format(self.max_processes))
            self.model.to('cpu')
            with Pool(self.max_processes) as p:
                tokenized_texts = list(p.imap(self.tokenize_example, examples, chunksize=self.chunk_size))
        for ex_index, example in enumerate(tqdm(examples, desc="Convert dataset")):
            if label_type is None:
                if isinstance(example.label, int):
                    label_type = torch.long
                elif isinstance(example.label, float):
                    label_type = torch.float
            tokenized_text = tokenized_texts[ex_index][0]
            if hasattr(model, 'max_seq_length') and model.max_seq_length is not None and model.max_seq_length > 0 and len(tokenized_text) > model.max_seq_length:
                too_long += 1
            if example.label in label_sent_mapping:
                label_sent_mapping[example.label].append(ex_index)
            else:
                label_sent_mapping[example.label] = [ex_index]
            inputs.append(tokenized_text)
            labels.append(example.label)
        distinct_labels = list(label_sent_mapping.keys())
        for i in range(len(distinct_labels)):
            label = distinct_labels[i]
            if len(label_sent_mapping[label]) >= 2:
                self.grouped_inputs.extend([inputs[j] for j in label_sent_mapping[label]])
                self.grouped_labels.extend([labels[j] for j in label_sent_mapping[label]])
                self.groups_right_border.append(len(self.grouped_inputs)) 
                self.num_labels += 1
        self.grouped_labels = torch.tensor(self.grouped_labels, dtype=label_type)
        logging.info("Num sentences: %d" % (len(self.grouped_inputs)))
        logging.info("Sentences longer than max_seqence_length: {}".format(too_long))
        logging.info("Number of labels with >1 examples: {}".format(len(distinct_labels)))
def convert_input_examples ( self , examples : List [ InputExample ] , model : SentenceTransformer ) : inputs = [ ] labels = [ ] label_sent_mapping = { } too_long = 0 label_type = None logging . info ( "Start tokenization" ) if not self . parallel_tokenization or self . max_processes == 1 or len ( examples ) <= self . chunk_size : tokenized_texts = [ self . tokenize_example ( example ) for example in examples ] else : logging . info ( "Use multi-process tokenization with {} processes" . format ( self . max_processes ) ) self . model . to ( 'cpu' ) with Pool ( self . max_processes ) as p : tokenized_texts = list ( p . imap ( self . tokenize_example , examples , chunksize = self . chunk_size ) ) for ex_index , example in enumerate ( tqdm ( examples , desc = "Convert dataset" ) ) : if label_type is None : if isinstance ( example . label , int ) : label_type = torch . long elif isinstance ( example . label , float ) : label_type = torch . float tokenized_text = tokenized_texts [ ex_index ] [ 0 ] if hasattr ( model , 'max_seq_length' ) and model . max_seq_length is not None and model . max_seq_length > 0 and len ( tokenized_text ) > model . max_seq_length : too_long += 1 if example . label in label_sent_mapping : label_sent_mapping [ example . label ] . append ( ex_index ) else : label_sent_mapping [ example . label ] = [ ex_index ] inputs . append ( tokenized_text ) labels . append ( example . label ) distinct_labels = list ( label_sent_mapping . keys ( ) ) for i in range ( len ( distinct_labels ) ) : label = distinct_labels [ i ] if len ( label_sent_mapping [ label ] ) >= 2 : self . grouped_inputs . extend ( [ inputs [ j ] for j in label_sent_mapping [ label ] ] ) self . grouped_labels . extend ( [ labels [ j ] for j in label_sent_mapping [ label ] ] ) self . groups_right_border . append ( len ( self . grouped_inputs ) ) self . num_labels += 1 self . grouped_labels = torch . tensor ( self . grouped_labels , dtype = label_type ) logging . info ( "Num sentences: %d" % ( len ( self . grouped_inputs ) ) ) logging . info ( "Sentences longer than max_seqence_length: {}" . format ( too_long ) ) logging . info ( "Number of labels with >1 examples: {}" . format ( len ( distinct_labels ) ) )
Converts input examples to a SentenceLabelDataset.
Converts input examples to a SentenceLabelDataset .
"""
        Converts input examples to a SentenceLabelDataset.

        Assumes only one sentence per InputExample and labels as integers from 0 to max_num_labels
        and should be used in combination with dataset_reader.LabelSentenceReader.

        Labels with only one example are ignored.

        :param examples:
            the input examples for the training
        :param model
            the Sentence Transformer model for the conversion
        :param is_pretokenized
            If set to true, no tokenization will be applied. It is expected that the input is tokenized via model.tokenize
        """ # Group examples and labels # Add examples with the same label to the same dict # Group sentences, such that sentences with the same label # are besides each other. Only take labels with at least 2 examples #At which position does this label group / bucket end?
self
examples
List[InputExample]
model
SentenceTransformer