algo/transformers/run_model.py
def load_and_cache_examples(
        self, examples, evaluate=False, no_cache=False, multi_label=False, verbose=True, silent=False
    ):
        """
        Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures.

        Utility function for train() and eval() methods. Not intended to be used directly.
        """

        process_count = self.args["process_count"]

        tokenizer = self.tokenizer
        args = self.args

        if not no_cache:
            no_cache = args["no_cache"]

        if not multi_label and args["regression"]:
            output_mode = "regression"
        else:
            output_mode = "classification"

        os.makedirs(self.args["cache_dir"], exist_ok=True)

        mode = "dev" if evaluate else "train"
        cached_features_file = os.path.join(
            args["cache_dir"],
            "cached_{}_{}_{}_{}_{}".format(
                mode, args["model_type"], args["max_seq_length"], self.num_labels, len(examples),
            ),
        )

        if os.path.exists(cached_features_file) and (
            (not args["reprocess_input_data"] and not no_cache) or (
                mode == "dev" and args["use_cached_eval_features"] and not no_cache)
        ):
            features = torch.load(cached_features_file)
            if verbose:
                print(f"Features loaded from cache at {cached_features_file}")
        else:
            if verbose:
                print(f"Converting to features started. Cache is not used.")
                if args["sliding_window"]:
                    print("Sliding window enabled")
            features = convert_examples_to_features(
                examples,
                args["max_seq_length"],
                tokenizer,
                output_mode,
                # XLNet has a CLS token at the end
                cls_token_at_end=bool(args["model_type"] in ["xlnet"]),
                cls_token=tokenizer.cls_token,
                cls_token_segment_id=2 if args["model_type"] in ["xlnet"] else 0,
                sep_token=tokenizer.sep_token,
                # RoBERTa uses an extra separator b/w pairs of sentences,
                # cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
                sep_token_extra=bool(args["model_type"] in ["roberta", "camembert", "xlmroberta"]),
                # PAD on the left for XLNet
                pad_on_left=bool(args["model_type"] in ["xlnet"]),
                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
                pad_token_segment_id=4 if args["model_type"] in ["xlnet"] else 0,
                process_count=process_count,
                multi_label=multi_label,
                silent=args["silent"] or silent,
                use_multiprocessing=args["use_multiprocessing"],
                sliding_window=args["sliding_window"],
                flatten=not evaluate,
                stride=args["stride"],
            )
            if verbose and args["sliding_window"]:
                print(f"{len(features)} features created from {len(examples)} samples.")

            if not no_cache:
                torch.save(features, cached_features_file)

        if args["sliding_window"] and evaluate:
            window_counts = [len(sample) for sample in features]
            features = [feature for feature_set in features for feature in feature_set]

        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)

        if output_mode == "classification":
            all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
        elif output_mode == "regression":
            all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)

        dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)

        if args["sliding_window"] and evaluate:
            return dataset, window_counts
        else:
            return dataset

        Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures.

        Utility function for train() and eval() methods. Not intended to be used directly.
        
Converts a list of InputExample objects to a TensorDataset containing InputFeatures.
Converts a list of InputExample objects to a TensorDataset containing InputFeatures .
def load_and_cache_examples(
        self, examples, evaluate=False, no_cache=False, multi_label=False, verbose=True, silent=False
    ):
        process_count = self.args["process_count"]
        tokenizer = self.tokenizer
        args = self.args
        if not no_cache:
            no_cache = args["no_cache"]
        if not multi_label and args["regression"]:
            output_mode = "regression"
        else:
            output_mode = "classification"
        os.makedirs(self.args["cache_dir"], exist_ok=True)
        mode = "dev" if evaluate else "train"
        cached_features_file = os.path.join(
            args["cache_dir"],
            "cached_{}_{}_{}_{}_{}".format(
                mode, args["model_type"], args["max_seq_length"], self.num_labels, len(examples),
            ),
        )
        if os.path.exists(cached_features_file) and (
            (not args["reprocess_input_data"] and not no_cache) or (
                mode == "dev" and args["use_cached_eval_features"] and not no_cache)
        ):
            features = torch.load(cached_features_file)
            if verbose:
                print(f"Features loaded from cache at {cached_features_file}")
        else:
            if verbose:
                print(f"Converting to features started. Cache is not used.")
                if args["sliding_window"]:
                    print("Sliding window enabled")
            features = convert_examples_to_features(
                examples,
                args["max_seq_length"],
                tokenizer,
                output_mode,
                cls_token_at_end=bool(args["model_type"] in ["xlnet"]),
                cls_token=tokenizer.cls_token,
                cls_token_segment_id=2 if args["model_type"] in ["xlnet"] else 0,
                sep_token=tokenizer.sep_token,
                sep_token_extra=bool(args["model_type"] in ["roberta", "camembert", "xlmroberta"]),
                pad_on_left=bool(args["model_type"] in ["xlnet"]),
                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
                pad_token_segment_id=4 if args["model_type"] in ["xlnet"] else 0,
                process_count=process_count,
                multi_label=multi_label,
                silent=args["silent"] or silent,
                use_multiprocessing=args["use_multiprocessing"],
                sliding_window=args["sliding_window"],
                flatten=not evaluate,
                stride=args["stride"],
            )
            if verbose and args["sliding_window"]:
                print(f"{len(features)} features created from {len(examples)} samples.")
            if not no_cache:
                torch.save(features, cached_features_file)
        if args["sliding_window"] and evaluate:
            window_counts = [len(sample) for sample in features]
            features = [feature for feature_set in features for feature in feature_set]
        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
        if output_mode == "classification":
            all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
        elif output_mode == "regression":
            all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)
        dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
        if args["sliding_window"] and evaluate:
            return dataset, window_counts
        else:
            return dataset
def load_and_cache_examples ( self , examples , evaluate = False , no_cache = False , multi_label = False , verbose = True , silent = False ) : process_count = self . args [ "process_count" ] tokenizer = self . tokenizer args = self . args if not no_cache : no_cache = args [ "no_cache" ] if not multi_label and args [ "regression" ] : output_mode = "regression" else : output_mode = "classification" os . makedirs ( self . args [ "cache_dir" ] , exist_ok = True ) mode = "dev" if evaluate else "train" cached_features_file = os . path . join ( args [ "cache_dir" ] , "cached_{}_{}_{}_{}_{}" . format ( mode , args [ "model_type" ] , args [ "max_seq_length" ] , self . num_labels , len ( examples ) , ) , ) if os . path . exists ( cached_features_file ) and ( ( not args [ "reprocess_input_data" ] and not no_cache ) or ( mode == "dev" and args [ "use_cached_eval_features" ] and not no_cache ) ) : features = torch . load ( cached_features_file ) if verbose : print ( f"Features loaded from cache at {cached_features_file}" ) else : if verbose : print ( f"Converting to features started. Cache is not used." ) if args [ "sliding_window" ] : print ( "Sliding window enabled" ) features = convert_examples_to_features ( examples , args [ "max_seq_length" ] , tokenizer , output_mode , cls_token_at_end = bool ( args [ "model_type" ] in [ "xlnet" ] ) , cls_token = tokenizer . cls_token , cls_token_segment_id = 2 if args [ "model_type" ] in [ "xlnet" ] else 0 , sep_token = tokenizer . sep_token , sep_token_extra = bool ( args [ "model_type" ] in [ "roberta" , "camembert" , "xlmroberta" ] ) , pad_on_left = bool ( args [ "model_type" ] in [ "xlnet" ] ) , pad_token = tokenizer . convert_tokens_to_ids ( [ tokenizer . pad_token ] ) [ 0 ] , pad_token_segment_id = 4 if args [ "model_type" ] in [ "xlnet" ] else 0 , process_count = process_count , multi_label = multi_label , silent = args [ "silent" ] or silent , use_multiprocessing = args [ "use_multiprocessing" ] , sliding_window = args [ "sliding_window" ] , flatten = not evaluate , stride = args [ "stride" ] , ) if verbose and args [ "sliding_window" ] : print ( f"{len(features)} features created from {len(examples)} samples." ) if not no_cache : torch . save ( features , cached_features_file ) if args [ "sliding_window" ] and evaluate : window_counts = [ len ( sample ) for sample in features ] features = [ feature for feature_set in features for feature in feature_set ] all_input_ids = torch . tensor ( [ f . input_ids for f in features ] , dtype = torch . long ) all_input_mask = torch . tensor ( [ f . input_mask for f in features ] , dtype = torch . long ) all_segment_ids = torch . tensor ( [ f . segment_ids for f in features ] , dtype = torch . long ) if output_mode == "classification" : all_label_ids = torch . tensor ( [ f . label_id for f in features ] , dtype = torch . long ) elif output_mode == "regression" : all_label_ids = torch . tensor ( [ f . label_id for f in features ] , dtype = torch . float ) dataset = TensorDataset ( all_input_ids , all_input_mask , all_segment_ids , all_label_ids ) if args [ "sliding_window" ] and evaluate : return dataset , window_counts else : return dataset
Converts a list of InputExample objects to a TensorDataset containing InputFeatures.
Converts a list of InputExample objects to a TensorDataset containing InputFeatures .
"""
        Converts a list of InputExample objects to a TensorDataset containing InputFeatures. Caches the InputFeatures.

        Utility function for train() and eval() methods. Not intended to be used directly.
        """ # XLNet has a CLS token at the end # RoBERTa uses an extra separator b/w pairs of sentences, # cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 # PAD on the left for XLNet
self
examples
evaluate
no_cache
multi_label
verbose
silent