# embeddings_queue

created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner.py:216-641

metadata:
{"docstring": "", "end_line": 641, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner.py", "name": "main", "start_line": 216, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_ner\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir)\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    if training_args.do_train:\n        column_names = raw_datasets[\"train\"].column_names\n        features = raw_datasets[\"train\"].features\n    else:\n        column_names = raw_datasets[\"validation\"].column_names\n        features = raw_datasets[\"validation\"].features\n\n    if data_args.text_column_name is not None:\n        text_column_name = data_args.text_column_name\n    elif \"tokens\" in column_names:\n        text_column_name = \"tokens\"\n    else:\n        text_column_name = column_names[0]\n\n    if data_args.label_column_name is not None:\n        label_column_name = data_args.label_column_name\n    elif f\"{data_args.task_name}_tags\" in column_names:\n        label_column_name = f\"{data_args.task_name}_tags\"\n    else:\n        label_column_name = column_names[1]\n\n    # In the event the labels are not a `Sequence[ClassLabel]`, we will need to go through the dataset to get the\n    # unique labels.\n    def get_label_list(labels):\n        unique_labels = set()\n        for label in labels:\n            unique_labels = unique_labels | set(label)\n        label_list = list(unique_labels)\n        label_list.sort()\n        return label_list\n\n    # If the labels are of type ClassLabel, they are already integers and we have the map stored somewhere.\n    # Otherwise, we have to get the list of labels manually.\n    labels_are_int = isinstance(features[label_column_name].feature, ClassLabel)\n    if labels_are_int:\n        label_list = features[label_column_name].feature.names\n        label_to_id = {i: i for i in range(len(label_list))}\n    else:\n        label_list = get_label_list(raw_datasets[\"train\"][label_column_name])\n        label_to_id = {l: i for i, l in enumerate(label_list)}\n\n    num_labels = len(label_list)\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        num_labels=num_labels,\n        finetuning_task=data_args.task_name,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path\n    if config.model_type in {\"bloom\", \"gpt2\", \"roberta\", \"deberta\"}:\n        tokenizer = AutoTokenizer.from_pretrained(\n            tokenizer_name_or_path,\n            cache_dir=model_args.cache_dir,\n            use_fast=True,\n            revision=model_args.model_revision,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n            add_prefix_space=True,\n        )\n    else:\n        tokenizer = AutoTokenizer.from_pretrained(\n            tokenizer_name_or_path,\n            cache_dir=model_args.cache_dir,\n            use_fast=True,\n            revision=model_args.model_revision,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n\n    model = AutoModelForTokenClassification.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n        ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,\n    )\n\n    # Tokenizer check: this script requires a fast tokenizer.\n    if not isinstance(tokenizer, PreTrainedTokenizerFast):\n        raise ValueError(\n            \"This example script only works for models that have a fast tokenizer. Checkout the big table of models at\"\n            \" https://huggingface.co/transformers/index.html#supported-frameworks to find the model types that meet\"\n            \" this requirement\"\n        )\n\n    # Model has labels -> use them.\n    if model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id:\n        if sorted(model.config.label2id.keys()) == sorted(label_list):\n            # Reorganize `label_list` to match the ordering of the model.\n            if labels_are_int:\n                label_to_id = {i: int(model.config.label2id[l]) for i, l in enumerate(label_list)}\n                label_list = [model.config.id2label[i] for i in range(num_labels)]\n            else:\n                label_list = [model.config.id2label[i] for i in range(num_labels)]\n                label_to_id = {l: i for i, l in enumerate(label_list)}\n        else:\n            logger.warning(\n                \"Your model seems to have been trained with labels, but they don't match the dataset: \"\n                f\"model labels: {sorted(model.config.label2id.keys())}, dataset labels:\"\n                f\" {sorted(label_list)}.\\nIgnoring the model labels as a result.\",\n            )\n\n    # Set the correspondences label/ID inside the model config\n    model.config.label2id = {l: i for i, l in enumerate(label_list)}\n    model.config.id2label = dict(enumerate(label_list))\n\n    # Map that sends B-Xxx label to its I-Xxx counterpart\n    b_to_i_label = []\n    for idx, label in enumerate(label_list):\n        if label.startswith(\"B-\") and label.replace(\"B-\", \"I-\") in label_list:\n            b_to_i_label.append(label_list.index(label.replace(\"B-\", \"I-\")))\n        else:\n            b_to_i_label.append(idx)\n\n    # Preprocessing the dataset\n    # Padding strategy\n    padding = \"max_length\" if data_args.pad_to_max_length else False\n\n    # Tokenize all texts and align the labels with them.\n    def tokenize_and_align_labels(examples):\n        tokenized_inputs = tokenizer(\n            examples[text_column_name],\n            padding=padding,\n            truncation=True,\n            max_length=data_args.max_seq_length,\n            # We use this argument because the texts in our dataset are lists of words (with a label for each word).\n            is_split_into_words=True,\n        )\n        labels = []\n        for i, label in enumerate(examples[label_column_name]):\n            word_ids = tokenized_inputs.word_ids(batch_index=i)\n            previous_word_idx = None\n            label_ids = []\n            for word_idx in word_ids:\n                # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n                # ignored in the loss function.\n                if word_idx is None:\n                    label_ids.append(-100)\n                # We set the label for the first token of each word.\n                elif word_idx != previous_word_idx:\n                    label_ids.append(label_to_id[label[word_idx]])\n                # For the other tokens in a word, we set the label to either the current label or -100, depending on\n                # the label_all_tokens flag.\n                else:\n                    if data_args.label_all_tokens:\n                        label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])\n                    else:\n                        label_ids.append(-100)\n                previous_word_idx = word_idx\n\n            labels.append(label_ids)\n        tokenized_inputs[\"labels\"] = labels\n        return tokenized_inputs\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                tokenize_and_align_labels,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n\n    if training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_dataset = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_dataset.map(\n                tokenize_and_align_labels,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n\n    if training_args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_dataset = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_dataset.map(\n                tokenize_and_align_labels,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n\n    # Data collator\n    data_collator = DataCollatorForTokenClassification(tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None)\n\n    # Metrics\n    metric = evaluate.load(\"seqeval\", cache_dir=model_args.cache_dir)\n\n    def compute_metrics(p):\n        predictions, labels = p\n        predictions = np.argmax(predictions, axis=2)\n\n        # Remove ignored index (special tokens)\n        true_predictions = [\n            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n            for prediction, label in zip(predictions, labels)\n        ]\n        true_labels = [\n            [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n            for prediction, label in zip(predictions, labels)\n        ]\n\n        results = metric.compute(predictions=true_predictions, references=true_labels)\n        if data_args.return_entity_level_metrics:\n            # Unpack nested dictionaries\n            final_results = {}\n            for key, value in results.items():\n                if isinstance(value, dict):\n                    for n, v in value.items():\n                        final_results[f\"{key}_{n}\"] = v\n                else:\n                    final_results[key] = value\n            return final_results\n        else:\n            return {\n                \"precision\": results[\"overall_precision\"],\n                \"recall\": results[\"overall_recall\"],\n                \"f1\": results[\"overall_f1\"],\n                \"accuracy\": results[\"overall_accuracy\"],\n            }\n\n    # Initialize our Trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        metrics = train_result.metrics\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n\n        metrics = trainer.evaluate()\n\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Predict\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n\n        predictions, labels, metrics = trainer.predict(predict_dataset, metric_key_prefix=\"predict\")\n        predictions = np.argmax(predictions, axis=2)\n\n        # Remove ignored index (special tokens)\n        true_predictions = [\n            [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n            for prediction, label in zip(predictions, labels)\n        ]\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n        # Save predictions\n        output_predictions_file = os.path.join(training_args.output_dir, \"predictions.txt\")\n        if trainer.is_world_process_zero():\n            with open(output_predictions_file, \"w\") as writer:\n                for prediction in true_predictions:\n                    writer.write(\" \".join(prediction) + \"\\n\")\n\n    kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"token-classification\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nThis code is for training and testing a Hugging Face model, specifically a sequence classification model, on a token classification task. The model is trained on a dataset provided in the `data_args` object.\n\nMain components of this code:\n\n1. **Data**: The data is loaded using the `Tokenizer` class from Hugging Face. The data is tokenized and labeled.\n2. **Model**: The model is instantiated with the specified parameters, including the learning rate, batch size, and number of epochs.\n3. **Trainer**: The `Trainer` class from Hugging Face is used to train the model on the training dataset. The trainer also evaluates the model on the evaluation dataset and predicts on the prediction dataset.\n4. **Evaluation Metrics**: Several metrics are computed during the training process, including precision, recall, F1-score, accuracy, and entity-level metrics.\n\nThe code also includes features for:\n\n* Resuming from a checkpoint\n* Logging metrics to a file\n* Saving predictions to a file\n* Pushing the model to Hugging Face's Model Hub\n\nOverall, this code provides a basic framework for training and evaluating a sequence classification model using Hugging Face."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner.py:644-646

metadata:
{"docstring": "", "end_line": 646, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner.py", "name": "_mp_fn", "start_line": 644, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Function Summary:**\n\n`_mp_fn(index)` is an internal control flow function used with XLA Spawn (TPU) infrastructure. Its primary purpose is to execute the `main()` function.\n\n**Key Implementation Details:**\n\n* Expects an `index` parameter but does not utilize it.\n* Calls `main()`, which is not part of this function's implementation.\n\n**Notable Dependencies or Requirements:**\n\n* Implicitly relies on XLA Spawn (TPU) infrastructure for execution."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner_no_trainer.py:68-269

metadata:
{"docstring": "", "end_line": 269, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner_no_trainer.py", "name": "parse_args", "start_line": 68, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(\n        description=\"Finetune a transformers model on a text classification task (NER) with accelerate library\"\n    )\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--text_column_name\",\n        type=str,\n        default=None,\n        help=\"The column name of text to input in the file (a csv or JSON file).\",\n    )\n    parser.add_argument(\n        \"--label_column_name\",\n        type=str,\n        default=None,\n        help=\"The column name of label to input in the file (a csv or JSON file).\",\n    )\n    parser.add_argument(\n        \"--max_length\",\n        type=int,\n        default=128,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,\"\n            \" sequences shorter will be padded if `--pad_to_max_length` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\n        \"--label_all_tokens\",\n        action=\"store_true\",\n        help=\"Setting labels of all special tokens to -100 and thus PyTorch will ignore them.\",\n    )\n    parser.add_argument(\n        \"--return_entity_level_metrics\",\n        action=\"store_true\",\n        help=\"Indication whether entity level metrics are to be returner.\",\n    )\n    parser.add_argument(\n        \"--task_name\",\n        type=str,\n        default=\"ner\",\n        choices=[\"ner\", \"pos\", \"chunk\"],\n        help=\"The name of the task.\",\n    )\n    parser.add_argument(\n        \"--debug\",\n        action=\"store_true\",\n        help=\"Activate debug mode and run training only with a subset of data.\",\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--ignore_mismatched_sizes\",\n        action=\"store_true\",\n        help=\"Whether or not to enable to load a pretrained model whose head dimensions are different.\",\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.task_name is None and args.train_file is None and args.validation_file is None:\n        raise ValueError(\"Need either a task name or a training/validation file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\n**Summary**\n\nThis function defines the command-line interface for fine-tuning a transformer model on a text classification task (NER) with the Accelerate library. It handles various input parameters, including dataset names, configuration names, training and validation files, model name or path, hyperparameters, and more.\n\n**Key Implementation Details**\n\n*   Parses command-line arguments using `argparse.ArgumentParser`.\n*   Expects various file extensions for training and validation data.\n*   Handles hyperparameters such as learning rate, batch size, number of epochs, and others.\n*   Supports fine-tuning from a pre-trained model or model identifier.\n\n**Notable Dependencies/Requirements**\n\n*   `argparse` library for command-line argument parsing\n*   Accelerate library for fine-tuning transformer models\n*   Datasets library for handling dataset names and configuration names\n*   Hugging Face transformers library (optional)"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner_no_trainer.py:272-827

metadata:
{"docstring": "", "end_line": 827, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/token-classification/run_ner_no_trainer.py", "name": "main", "start_line": 272, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_ner_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator = (\n        Accelerator(log_with=args.report_to, project_dir=args.output_dir) if args.with_tracking else Accelerator()\n    )\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets for token classification task available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'tokens' or the first column if no column called\n    # 'tokens' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files)\n    # Trim a number of training examples\n    if args.debug:\n        for split in raw_datasets.keys():\n            raw_datasets[split] = raw_datasets[split].select(range(100))\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    if raw_datasets[\"train\"] is not None:\n        column_names = raw_datasets[\"train\"].column_names\n        features = raw_datasets[\"train\"].features\n    else:\n        column_names = raw_datasets[\"validation\"].column_names\n        features = raw_datasets[\"validation\"].features\n\n    if args.text_column_name is not None:\n        text_column_name = args.text_column_name\n    elif \"tokens\" in column_names:\n        text_column_name = \"tokens\"\n    else:\n        text_column_name = column_names[0]\n\n    if args.label_column_name is not None:\n        label_column_name = args.label_column_name\n    elif f\"{args.task_name}_tags\" in column_names:\n        label_column_name = f\"{args.task_name}_tags\"\n    else:\n        label_column_name = column_names[1]\n\n    # In the event the labels are not a `Sequence[ClassLabel]`, we will need to go through the dataset to get the\n    # unique labels.\n    def get_label_list(labels):\n        unique_labels = set()\n        for label in labels:\n            unique_labels = unique_labels | set(label)\n        label_list = list(unique_labels)\n        label_list.sort()\n        return label_list\n\n    # If the labels are of type ClassLabel, they are already integers and we have the map stored somewhere.\n    # Otherwise, we have to get the list of labels manually.\n    labels_are_int = isinstance(features[label_column_name].feature, ClassLabel)\n    if labels_are_int:\n        label_list = features[label_column_name].feature.names\n        label_to_id = {i: i for i in range(len(label_list))}\n    else:\n        label_list = get_label_list(raw_datasets[\"train\"][label_column_name])\n        label_to_id = {l: i for i, l in enumerate(label_list)}\n\n    num_labels = len(label_list)\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(\n            args.config_name, num_labels=num_labels, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(\n            args.model_name_or_path, num_labels=num_labels, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    tokenizer_name_or_path = args.tokenizer_name if args.tokenizer_name else args.model_name_or_path\n    if not tokenizer_name_or_path:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if config.model_type in {\"bloom\", \"gpt2\", \"roberta\"}:\n        tokenizer = AutoTokenizer.from_pretrained(\n            tokenizer_name_or_path, use_fast=True, add_prefix_space=True, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        tokenizer = AutoTokenizer.from_pretrained(\n            tokenizer_name_or_path, use_fast=True, trust_remote_code=args.trust_remote_code\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForTokenClassification.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            ignore_mismatched_sizes=args.ignore_mismatched_sizes,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForTokenClassification.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    # Model has labels -> use them.\n    if model.config.label2id != PretrainedConfig(num_labels=num_labels).label2id:\n        if sorted(model.config.label2id.keys()) == sorted(label_list):\n            # Reorganize `label_list` to match the ordering of the model.\n            if labels_are_int:\n                label_to_id = {i: int(model.config.label2id[l]) for i, l in enumerate(label_list)}\n                label_list = [model.config.id2label[i] for i in range(num_labels)]\n            else:\n                label_list = [model.config.id2label[i] for i in range(num_labels)]\n                label_to_id = {l: i for i, l in enumerate(label_list)}\n        else:\n            logger.warning(\n                \"Your model seems to have been trained with labels, but they don't match the dataset: \"\n                f\"model labels: {sorted(model.config.label2id.keys())}, dataset labels:\"\n                f\" {sorted(label_list)}.\\nIgnoring the model labels as a result.\",\n            )\n\n    # Set the correspondences label/ID inside the model config\n    model.config.label2id = {l: i for i, l in enumerate(label_list)}\n    model.config.id2label = dict(enumerate(label_list))\n\n    # Map that sends B-Xxx label to its I-Xxx counterpart\n    b_to_i_label = []\n    for idx, label in enumerate(label_list):\n        if label.startswith(\"B-\") and label.replace(\"B-\", \"I-\") in label_list:\n            b_to_i_label.append(label_list.index(label.replace(\"B-\", \"I-\")))\n        else:\n            b_to_i_label.append(idx)\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    padding = \"max_length\" if args.pad_to_max_length else False\n\n    # Tokenize all texts and align the labels with them.\n\n    def tokenize_and_align_labels(examples):\n        tokenized_inputs = tokenizer(\n            examples[text_column_name],\n            max_length=args.max_length,\n            padding=padding,\n            truncation=True,\n            # We use this argument because the texts in our dataset are lists of words (with a label for each word).\n            is_split_into_words=True,\n        )\n\n        labels = []\n        for i, label in enumerate(examples[label_column_name]):\n            word_ids = tokenized_inputs.word_ids(batch_index=i)\n            previous_word_idx = None\n            label_ids = []\n            for word_idx in word_ids:\n                # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n                # ignored in the loss function.\n                if word_idx is None:\n                    label_ids.append(-100)\n                # We set the label for the first token of each word.\n                elif word_idx != previous_word_idx:\n                    label_ids.append(label_to_id[label[word_idx]])\n                # For the other tokens in a word, we set the label to either the current label or -100, depending on\n                # the label_all_tokens flag.\n                else:\n                    if args.label_all_tokens:\n                        label_ids.append(b_to_i_label[label_to_id[label[word_idx]]])\n                    else:\n                        label_ids.append(-100)\n                previous_word_idx = word_idx\n\n            labels.append(label_ids)\n        tokenized_inputs[\"labels\"] = labels\n        return tokenized_inputs\n\n    with accelerator.main_process_first():\n        processed_raw_datasets = raw_datasets.map(\n            tokenize_and_align_labels,\n            batched=True,\n            remove_columns=raw_datasets[\"train\"].column_names,\n            desc=\"Running tokenizer on dataset\",\n        )\n\n    train_dataset = processed_raw_datasets[\"train\"]\n    eval_dataset = processed_raw_datasets[\"validation\"]\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    if args.pad_to_max_length:\n        # If padding was already done ot max length, we use the default data collator that will just convert everything\n        # to tensors.\n        data_collator = default_data_collator\n    else:\n        # Otherwise, `DataCollatorForTokenClassification` will apply dynamic padding for us (by padding to the maximum length of\n        # the samples passed). When using mixed precision, we add `pad_to_multiple_of=8` to pad all tensors to multiple\n        # of 8s, which will enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\n        # For fp8, we pad to multiple of 16.\n        if accelerator.mixed_precision == \"fp8\":\n            pad_to_multiple_of = 16\n        elif accelerator.mixed_precision != \"no\":\n            pad_to_multiple_of = 8\n        else:\n            pad_to_multiple_of = None\n        data_collator = DataCollatorForTokenClassification(tokenizer, pad_to_multiple_of=pad_to_multiple_of)\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Use the device given by the `accelerator` object.\n    device = accelerator.device\n    model.to(device)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps,\n        num_training_steps=args.max_train_steps,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"ner_no_trainer\", experiment_config)\n\n    # Metrics\n    metric = evaluate.load(\"seqeval\")\n\n    def get_labels(predictions, references):\n        # Transform predictions and references tensos to numpy arrays\n        if device.type == \"cpu\":\n            y_pred = predictions.detach().clone().numpy()\n            y_true = references.detach().clone().numpy()\n        else:\n            y_pred = predictions.detach().cpu().clone().numpy()\n            y_true = references.detach().cpu().clone().numpy()\n\n        # Remove ignored index (special tokens)\n        true_predictions = [\n            [label_list[p] for (p, l) in zip(pred, gold_label) if l != -100]\n            for pred, gold_label in zip(y_pred, y_true)\n        ]\n        true_labels = [\n            [label_list[l] for (p, l) in zip(pred, gold_label) if l != -100]\n            for pred, gold_label in zip(y_pred, y_true)\n        ]\n        return true_predictions, true_labels\n\n    def compute_metrics():\n        results = metric.compute()\n        if args.return_entity_level_metrics:\n            # Unpack nested dictionaries\n            final_results = {}\n            for key, value in results.items():\n                if isinstance(value, dict):\n                    for n, v in value.items():\n                        final_results[f\"{key}_{n}\"] = v\n                else:\n                    final_results[key] = value\n            return final_results\n        else:\n            return {\n                \"precision\": results[\"overall_precision\"],\n                \"recall\": results[\"overall_recall\"],\n                \"f1\": results[\"overall_f1\"],\n                \"accuracy\": results[\"overall_accuracy\"],\n            }\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            outputs = model(**batch)\n            loss = outputs.loss\n            # We keep track of the loss at each epoch\n            if args.with_tracking:\n                total_loss += loss.detach().float()\n            loss = loss / args.gradient_accumulation_steps\n            accelerator.backward(loss)\n            if step % args.gradient_accumulation_steps == 0 or step == len(train_dataloader) - 1:\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n        samples_seen = 0\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n            predictions = outputs.logits.argmax(dim=-1)\n            labels = batch[\"labels\"]\n            if not args.pad_to_max_length:  # necessary to pad predictions and labels for being gathered\n                predictions = accelerator.pad_across_processes(predictions, dim=1, pad_index=-100)\n                labels = accelerator.pad_across_processes(labels, dim=1, pad_index=-100)\n            predictions_gathered, labels_gathered = accelerator.gather((predictions, labels))\n            # If we are in a multiprocess environment, the last batch has duplicates\n            if accelerator.num_processes > 1:\n                if step == len(eval_dataloader) - 1:\n                    predictions_gathered = predictions_gathered[: len(eval_dataloader.dataset) - samples_seen]\n                    labels_gathered = labels_gathered[: len(eval_dataloader.dataset) - samples_seen]\n                else:\n                    samples_seen += labels_gathered.shape[0]\n            preds, refs = get_labels(predictions_gathered, labels_gathered)\n            metric.add_batch(\n                predictions=preds,\n                references=refs,\n            )  # predictions and preferences are expected to be a nested list of labels, not label_ids\n\n        eval_metric = compute_metrics()\n        accelerator.print(f\"epoch {epoch}:\", eval_metric)\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"seqeval\": eval_metric,\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n            all_results = {f\"eval_{k}\": v for k, v in eval_metric.items()}\n            if args.with_tracking:\n                all_results.update({\"train_loss\": total_loss.item() / len(train_dataloader)})\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n                # Convert all float64 & int64 type numbers to float & int for json serialization\n                for key, value in all_results.items():\n                    if isinstance(value, np.float64):\n                        all_results[key] = float(value)\n                    elif isinstance(value, np.int64):\n                        all_results[key] = int(value)\n\n\nSUMMARY:\nThis is a training script for a NLP model using the Hugging Face library. The script trains a model on a specified dataset and saves the model to a directory. Here's a concise summary:\n\n**Key Features:**\n\n* Trains a NLP model on a specified dataset\n* Supports various metrics, including sequence evaluation (seqeval)\n* Allows for checkpointing at specific epochs or steps\n* Updates model weights using gradient descent with optimizer and learning rate scheduler\n* Saves the trained model to a directory\n* Pushes updates to Hugging Face model hub if requested\n\n**Code Organization:**\n\n* The script defines constants for hyperparameters, metrics, and output directories\n* It initializes the Hugging Face client and model loader\n* Training is performed in an iterative loop, where each iteration consists of:\n\t+ Forward pass through the model with a batch of input data\n\t+ Backward pass to update model weights using gradient descent\n\t+ Batch normalization and logging\n* After training concludes, the script saves the trained model to disk and pushes it to Hugging Face if requested. \n\nNote that this code appears to use some advanced features of the Hugging Face library, such as multi-processing, distributed training, and API calls. A more detailed explanation would require a separate response."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/utils_qa.py:31-249
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 249, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/utils_qa.py", "name": "postprocess_qa_predictions", "start_line": 31, "type": "function", "chroma:document": "CODE:\n\ndef postprocess_qa_predictions(\n    examples,\n    features,\n    predictions: Tuple[np.ndarray, np.ndarray],\n    version_2_with_negative: bool = False,\n    n_best_size: int = 20,\n    max_answer_length: int = 30,\n    null_score_diff_threshold: float = 0.0,\n    output_dir: Optional[str] = None,\n    prefix: Optional[str] = None,\n    log_level: Optional[int] = logging.WARNING,\n):\n    \"\"\"\n    Post-processes the predictions of a question-answering model to convert them to answers that are substrings of the\n    original contexts. This is the base postprocessing functions for models that only return start and end logits.\n\n    Args:\n        examples: The non-preprocessed dataset (see the main script for more information).\n        features: The processed dataset (see the main script for more information).\n        predictions (:obj:`Tuple[np.ndarray, np.ndarray]`):\n            The predictions of the model: two arrays containing the start logits and the end logits respectively. Its\n            first dimension must match the number of elements of :obj:`features`.\n        version_2_with_negative (:obj:`bool`, `optional`, defaults to :obj:`False`):\n            Whether or not the underlying dataset contains examples with no answers.\n        n_best_size (:obj:`int`, `optional`, defaults to 20):\n            The total number of n-best predictions to generate when looking for an answer.\n        max_answer_length (:obj:`int`, `optional`, defaults to 30):\n            The maximum length of an answer that can be generated. This is needed because the start and end predictions\n            are not conditioned on one another.\n        null_score_diff_threshold (:obj:`float`, `optional`, defaults to 0):\n            The threshold used to select the null answer: if the best answer has a score that is less than the score of\n            the null answer minus this threshold, the null answer is selected for this example (note that the score of\n            the null answer for an example giving several features is the minimum of the scores for the null answer on\n            each feature: all features must be aligned on the fact they `want` to predict a null answer).\n\n            Only useful when :obj:`version_2_with_negative` is :obj:`True`.\n        output_dir (:obj:`str`, `optional`):\n            If provided, the dictionaries of predictions, n_best predictions (with their scores and logits) and, if\n            :obj:`version_2_with_negative=True`, the dictionary of the scores differences between best and null\n            answers, are saved in `output_dir`.\n        prefix (:obj:`str`, `optional`):\n            If provided, the dictionaries mentioned above are saved with `prefix` added to their names.\n        log_level (:obj:`int`, `optional`, defaults to ``logging.WARNING``):\n            ``logging`` log level (e.g., ``logging.WARNING``)\n    \"\"\"\n    if len(predictions) != 2:\n        raise ValueError(\"`predictions` should be a tuple with two elements (start_logits, end_logits).\")\n    all_start_logits, all_end_logits = predictions\n\n    if len(predictions[0]) != len(features):\n        raise ValueError(f\"Got {len(predictions[0])} predictions and {len(features)} features.\")\n\n    # Build a map example to its corresponding features.\n    example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n    features_per_example = collections.defaultdict(list)\n    for i, feature in enumerate(features):\n        features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)\n\n    # The dictionaries we have to fill.\n    all_predictions = collections.OrderedDict()\n    all_nbest_json = collections.OrderedDict()\n    if version_2_with_negative:\n        scores_diff_json = collections.OrderedDict()\n\n    # Logging.\n    logger.setLevel(log_level)\n    logger.info(f\"Post-processing {len(examples)} example predictions split into {len(features)} features.\")\n\n    # Let's loop over all the examples!\n    for example_index, example in enumerate(tqdm(examples)):\n        # Those are the indices of the features associated to the current example.\n        feature_indices = features_per_example[example_index]\n\n        min_null_prediction = None\n        prelim_predictions = []\n\n        # Looping through all the features associated to the current example.\n        for feature_index in feature_indices:\n            # We grab the predictions of the model for this feature.\n            start_logits = all_start_logits[feature_index]\n            end_logits = all_end_logits[feature_index]\n            # This is what will allow us to map some the positions in our logits to span of texts in the original\n            # context.\n            offset_mapping = features[feature_index][\"offset_mapping\"]\n            # Optional `token_is_max_context`, if provided we will remove answers that do not have the maximum context\n            # available in the current feature.\n            token_is_max_context = features[feature_index].get(\"token_is_max_context\", None)\n\n            # Update minimum null prediction.\n            feature_null_score = start_logits[0] + end_logits[0]\n            if min_null_prediction is None or min_null_prediction[\"score\"] > feature_null_score:\n                min_null_prediction = {\n                    \"offsets\": (0, 0),\n                    \"score\": feature_null_score,\n                    \"start_logit\": start_logits[0],\n                    \"end_logit\": end_logits[0],\n                }\n\n            # Go through all possibilities for the `n_best_size` greater start and end logits.\n            start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n            end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n            for start_index in start_indexes:\n                for end_index in end_indexes:\n                    # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n                    # to part of the input_ids that are not in the context.\n                    if (\n                        start_index >= len(offset_mapping)\n                        or end_index >= len(offset_mapping)\n                        or offset_mapping[start_index] is None\n                        or len(offset_mapping[start_index]) < 2\n                        or offset_mapping[end_index] is None\n                        or len(offset_mapping[end_index]) < 2\n                    ):\n                        continue\n                    # Don't consider answers with a length that is either < 0 or > max_answer_length.\n                    if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n                        continue\n                    # Don't consider answer that don't have the maximum context available (if such information is\n                    # provided).\n                    if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):\n                        continue\n\n                    prelim_predictions.append(\n                        {\n                            \"offsets\": (offset_mapping[start_index][0], offset_mapping[end_index][1]),\n                            \"score\": start_logits[start_index] + end_logits[end_index],\n                            \"start_logit\": start_logits[start_index],\n                            \"end_logit\": end_logits[end_index],\n                        }\n                    )\n        if version_2_with_negative and min_null_prediction is not None:\n            # Add the minimum null prediction\n            prelim_predictions.append(min_null_prediction)\n            null_score = min_null_prediction[\"score\"]\n\n        # Only keep the best `n_best_size` predictions.\n        predictions = sorted(prelim_predictions, key=lambda x: x[\"score\"], reverse=True)[:n_best_size]\n\n        # Add back the minimum null prediction if it was removed because of its low score.\n        if (\n            version_2_with_negative\n            and min_null_prediction is not None\n            and not any(p[\"offsets\"] == (0, 0) for p in predictions)\n        ):\n            predictions.append(min_null_prediction)\n\n        # Use the offsets to gather the answer text in the original context.\n        context = example[\"context\"]\n        for pred in predictions:\n            offsets = pred.pop(\"offsets\")\n            pred[\"text\"] = context[offsets[0] : offsets[1]]\n\n        # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid\n        # failure.\n        if len(predictions) == 0 or (len(predictions) == 1 and predictions[0][\"text\"] == \"\"):\n            predictions.insert(0, {\"text\": \"empty\", \"start_logit\": 0.0, \"end_logit\": 0.0, \"score\": 0.0})\n\n        # Compute the softmax of all scores (we do it with numpy to stay independent from torch/tf in this file, using\n        # the LogSumExp trick).\n        scores = np.array([pred.pop(\"score\") for pred in predictions])\n        exp_scores = np.exp(scores - np.max(scores))\n        probs = exp_scores / exp_scores.sum()\n\n        # Include the probabilities in our predictions.\n        for prob, pred in zip(probs, predictions):\n            pred[\"probability\"] = prob\n\n        # Pick the best prediction. If the null answer is not possible, this is easy.\n        if not version_2_with_negative:\n            all_predictions[example[\"id\"]] = predictions[0][\"text\"]\n        else:\n            # Otherwise we first need to find the best non-empty prediction.\n            i = 0\n            while predictions[i][\"text\"] == \"\":\n                i += 1\n            best_non_null_pred = predictions[i]\n\n            # Then we compare to the null prediction using the threshold.\n            score_diff = null_score - best_non_null_pred[\"start_logit\"] - best_non_null_pred[\"end_logit\"]\n            scores_diff_json[example[\"id\"]] = float(score_diff)  # To be JSON-serializable.\n            if score_diff > null_score_diff_threshold:\n                all_predictions[example[\"id\"]] = \"\"\n            else:\n                all_predictions[example[\"id\"]] = best_non_null_pred[\"text\"]\n\n        # Make `predictions` JSON-serializable by casting np.float back to float.\n        all_nbest_json[example[\"id\"]] = [\n            {k: (float(v) if isinstance(v, (np.float16, np.float32, np.float64)) else v) for k, v in pred.items()}\n            for pred in predictions\n        ]\n\n    # If we have an output_dir, let's save all those dicts.\n    if output_dir is not None:\n        if not os.path.isdir(output_dir):\n            raise EnvironmentError(f\"{output_dir} is not a directory.\")\n\n        prediction_file = os.path.join(\n            output_dir, \"predictions.json\" if prefix is None else f\"{prefix}_predictions.json\"\n        )\n        nbest_file = os.path.join(\n            output_dir, \"nbest_predictions.json\" if prefix is None else f\"{prefix}_nbest_predictions.json\"\n        )\n        if version_2_with_negative:\n            null_odds_file = os.path.join(\n                output_dir, \"null_odds.json\" if prefix is None else f\"{prefix}_null_odds.json\"\n            )\n\n        logger.info(f\"Saving predictions to {prediction_file}.\")\n        with open(prediction_file, \"w\") as writer:\n            writer.write(json.dumps(all_predictions, indent=4) + \"\\n\")\n        logger.info(f\"Saving nbest_preds to {nbest_file}.\")\n        with open(nbest_file, \"w\") as writer:\n            writer.write(json.dumps(all_nbest_json, indent=4) + \"\\n\")\n        if version_2_with_negative:\n            logger.info(f\"Saving null_odds to {null_odds_file}.\")\n            with open(null_odds_file, \"w\") as writer:\n                writer.write(json.dumps(scores_diff_json, indent=4) + \"\\n\")\n\n\n\nSUMMARY:\nHere is a concise summary of the provided code:\n\n**Overview**\n\nThis library function generates text answers for questions based on their context and scores generated from machine learning models. It takes in several parameters, including:\n\n* `n_best_size`: the number of best predictions to return\n* `version_2_with_negative`: whether the model can output a null answer (i.e., a blank response)\n* `context`: the question or text being answered about\n* `predictions_file`, `nbest_predictions_file`, and `null_odds_file` optional parameters for saving the results\n\n**Functionality**\n\n1. The function generates predictions by iterating through each prediction in ascending order of score, excluding null answers (if applicable).\n2. For each prediction, it extracts the relevant text from the original context using offsets.\n3. It then computes the softmax of all scores and adds the corresponding probabilities to the prediction dictionary.\n4. If a null answer is returned, its format is determined by the `version_2_with_negative` parameter.\n5. The function returns a dictionary with the best predictions for each question.\n\n**Notes**\n\n* The code uses NumPy to perform some calculations, but it includes checks to ensure compatibility with different math libraries (e.g., PyTorch and TensorFlow).\n* The null answer handling is conditional on the `version_2_with_negative` parameter.\n* Some parts of the code (e.g., comment block) are not shown due to formatting constraints."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/utils_qa.py:252-443

metadata:
{"docstring": "", "end_line": 443, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/utils_qa.py", "name": "postprocess_qa_predictions_with_beam_search", "start_line": 252, "type": "function", "chroma:document": "CODE:\n\ndef postprocess_qa_predictions_with_beam_search(\n    examples,\n    features,\n    predictions: Tuple[np.ndarray, np.ndarray],\n    version_2_with_negative: bool = False,\n    n_best_size: int = 20,\n    max_answer_length: int = 30,\n    start_n_top: int = 5,\n    end_n_top: int = 5,\n    output_dir: Optional[str] = None,\n    prefix: Optional[str] = None,\n    log_level: Optional[int] = logging.WARNING,\n):\n    \"\"\"\n    Post-processes the predictions of a question-answering model with beam search to convert them to answers that are substrings of the\n    original contexts. This is the postprocessing functions for models that return start and end logits, indices, as well as\n    cls token predictions.\n\n    Args:\n        examples: The non-preprocessed dataset (see the main script for more information).\n        features: The processed dataset (see the main script for more information).\n        predictions (:obj:`Tuple[np.ndarray, np.ndarray]`):\n            The predictions of the model: two arrays containing the start logits and the end logits respectively. Its\n            first dimension must match the number of elements of :obj:`features`.\n        version_2_with_negative (:obj:`bool`, `optional`, defaults to :obj:`False`):\n            Whether or not the underlying dataset contains examples with no answers.\n        n_best_size (:obj:`int`, `optional`, defaults to 20):\n            The total number of n-best predictions to generate when looking for an answer.\n        max_answer_length (:obj:`int`, `optional`, defaults to 30):\n            The maximum length of an answer that can be generated. This is needed because the start and end predictions\n            are not conditioned on one another.\n        start_n_top (:obj:`int`, `optional`, defaults to 5):\n            The number of top start logits too keep when searching for the :obj:`n_best_size` predictions.\n        end_n_top (:obj:`int`, `optional`, defaults to 5):\n            The number of top end logits too keep when searching for the :obj:`n_best_size` predictions.\n        output_dir (:obj:`str`, `optional`):\n            If provided, the dictionaries of predictions, n_best predictions (with their scores and logits) and, if\n            :obj:`version_2_with_negative=True`, the dictionary of the scores differences between best and null\n            answers, are saved in `output_dir`.\n        prefix (:obj:`str`, `optional`):\n            If provided, the dictionaries mentioned above are saved with `prefix` added to their names.\n        log_level (:obj:`int`, `optional`, defaults to ``logging.WARNING``):\n            ``logging`` log level (e.g., ``logging.WARNING``)\n    \"\"\"\n    if len(predictions) != 5:\n        raise ValueError(\"`predictions` should be a tuple with five elements.\")\n    start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits = predictions\n\n    if len(predictions[0]) != len(features):\n        raise ValueError(f\"Got {len(predictions[0])} predictions and {len(features)} features.\")\n\n    # Build a map example to its corresponding features.\n    example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n    features_per_example = collections.defaultdict(list)\n    for i, feature in enumerate(features):\n        features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)\n\n    # The dictionaries we have to fill.\n    all_predictions = collections.OrderedDict()\n    all_nbest_json = collections.OrderedDict()\n    scores_diff_json = collections.OrderedDict() if version_2_with_negative else None\n\n    # Logging.\n    logger.setLevel(log_level)\n    logger.info(f\"Post-processing {len(examples)} example predictions split into {len(features)} features.\")\n\n    # Let's loop over all the examples!\n    for example_index, example in enumerate(tqdm(examples)):\n        # Those are the indices of the features associated to the current example.\n        feature_indices = features_per_example[example_index]\n\n        min_null_score = None\n        prelim_predictions = []\n\n        # Looping through all the features associated to the current example.\n        for feature_index in feature_indices:\n            # We grab the predictions of the model for this feature.\n            start_log_prob = start_top_log_probs[feature_index]\n            start_indexes = start_top_index[feature_index]\n            end_log_prob = end_top_log_probs[feature_index]\n            end_indexes = end_top_index[feature_index]\n            feature_null_score = cls_logits[feature_index]\n            # This is what will allow us to map some the positions in our logits to span of texts in the original\n            # context.\n            offset_mapping = features[feature_index][\"offset_mapping\"]\n            # Optional `token_is_max_context`, if provided we will remove answers that do not have the maximum context\n            # available in the current feature.\n            token_is_max_context = features[feature_index].get(\"token_is_max_context\", None)\n\n            # Update minimum null prediction\n            if min_null_score is None or feature_null_score < min_null_score:\n                min_null_score = feature_null_score\n\n            # Go through all possibilities for the `n_start_top`/`n_end_top` greater start and end logits.\n            for i in range(start_n_top):\n                for j in range(end_n_top):\n                    start_index = int(start_indexes[i])\n                    j_index = i * end_n_top + j\n                    end_index = int(end_indexes[j_index])\n                    # Don't consider out-of-scope answers (last part of the test should be unnecessary because of the\n                    # p_mask but let's not take any risk)\n                    if (\n                        start_index >= len(offset_mapping)\n                        or end_index >= len(offset_mapping)\n                        or offset_mapping[start_index] is None\n                        or len(offset_mapping[start_index]) < 2\n                        or offset_mapping[end_index] is None\n                        or len(offset_mapping[end_index]) < 2\n                    ):\n                        continue\n\n                    # Don't consider answers with a length negative or > max_answer_length.\n                    if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n                        continue\n                    # Don't consider answer that don't have the maximum context available (if such information is\n                    # provided).\n                    if token_is_max_context is not None and not token_is_max_context.get(str(start_index), False):\n                        continue\n                    prelim_predictions.append(\n                        {\n                            \"offsets\": (offset_mapping[start_index][0], offset_mapping[end_index][1]),\n                            \"score\": start_log_prob[i] + end_log_prob[j_index],\n                            \"start_log_prob\": start_log_prob[i],\n                            \"end_log_prob\": end_log_prob[j_index],\n                        }\n                    )\n\n        # Only keep the best `n_best_size` predictions.\n        predictions = sorted(prelim_predictions, key=lambda x: x[\"score\"], reverse=True)[:n_best_size]\n\n        # Use the offsets to gather the answer text in the original context.\n        context = example[\"context\"]\n        for pred in predictions:\n            offsets = pred.pop(\"offsets\")\n            pred[\"text\"] = context[offsets[0] : offsets[1]]\n\n        # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid\n        # failure.\n        if len(predictions) == 0:\n            # Without predictions min_null_score is going to be None and None will cause an exception later\n            min_null_score = -2e-6\n            predictions.insert(0, {\"text\": \"\", \"start_logit\": -1e-6, \"end_logit\": -1e-6, \"score\": min_null_score})\n\n        # Compute the softmax of all scores (we do it with numpy to stay independent from torch/tf in this file, using\n        # the LogSumExp trick).\n        scores = np.array([pred.pop(\"score\") for pred in predictions])\n        exp_scores = np.exp(scores - np.max(scores))\n        probs = exp_scores / exp_scores.sum()\n\n        # Include the probabilities in our predictions.\n        for prob, pred in zip(probs, predictions):\n            pred[\"probability\"] = prob\n\n        # Pick the best prediction and set the probability for the null answer.\n        all_predictions[example[\"id\"]] = predictions[0][\"text\"]\n        if version_2_with_negative:\n            scores_diff_json[example[\"id\"]] = float(min_null_score)\n\n        # Make `predictions` JSON-serializable by casting np.float back to float.\n        all_nbest_json[example[\"id\"]] = [\n            {k: (float(v) if isinstance(v, (np.float16, np.float32, np.float64)) else v) for k, v in pred.items()}\n            for pred in predictions\n        ]\n\n    # If we have an output_dir, let's save all those dicts.\n    if output_dir is not None:\n        if not os.path.isdir(output_dir):\n            raise EnvironmentError(f\"{output_dir} is not a directory.\")\n\n        prediction_file = os.path.join(\n            output_dir, \"predictions.json\" if prefix is None else f\"{prefix}_predictions.json\"\n        )\n        nbest_file = os.path.join(\n            output_dir, \"nbest_predictions.json\" if prefix is None else f\"{prefix}_nbest_predictions.json\"\n        )\n        if version_2_with_negative:\n            null_odds_file = os.path.join(\n                output_dir, \"null_odds.json\" if prefix is None else f\"{prefix}_null_odds.json\"\n            )\n\n        logger.info(f\"Saving predictions to {prediction_file}.\")\n        with open(prediction_file, \"w\") as writer:\n            writer.write(json.dumps(all_predictions, indent=4) + \"\\n\")\n        logger.info(f\"Saving nbest_preds to {nbest_file}.\")\n        with open(nbest_file, \"w\") as writer:\n            writer.write(json.dumps(all_nbest_json, indent=4) + \"\\n\")\n        if version_2_with_negative:\n            logger.info(f\"Saving null_odds to {null_odds_file}.\")\n            with open(null_odds_file, \"w\") as writer:\n                writer.write(json.dumps(scores_diff_json, indent=4) + \"\\n\")\n\n\n\nSUMMARY:\nHere is a concise summary:\n\nThis code implements a function `compute_answers` that generates answers to a question from a set of predictions. The predictions are derived from a question's context and the model's confidence scores for each possible answer. The function takes as input:\n\n* A list of questions with their corresponding contexts\n* A list of prediction objects, each containing:\n\t+ An offset range for the answer in the context\n\t+ A confidence score (derived from start and end log probabilities)\n\t+ A start log probability\n\t+ An end log probability (transformed using LogSumExp trick)\n* The maximum length of answers to consider\n* Optional output directory to save the predictions\n\nThe function returns two dictionaries:\n\n* `all_predictions`: a dictionary mapping each question ID to its best answer (i.e., the one with the highest confidence score)\n* `scores_diff_json` (optional): a dictionary mapping each question ID to their null odds score (value equal to -2e-6 in case no prediction is found)\n\nThe code uses NumPy for computation and JSON serialization/deserialization."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/trainer_qa.py:30-136

metadata:
{"docstring": "", "end_line": 136, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/trainer_qa.py", "name": "QuestionAnsweringTrainer", "start_line": 30, "type": "class", "chroma:document": "CODE:\n\nclass QuestionAnsweringTrainer(Trainer):\n    def __init__(self, *args, eval_examples=None, post_process_function=None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.eval_examples = eval_examples\n        self.post_process_function = post_process_function\n\n    def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None, metric_key_prefix: str = \"eval\"):\n        eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset\n        eval_dataloader = self.get_eval_dataloader(eval_dataset)\n        eval_examples = self.eval_examples if eval_examples is None else eval_examples\n\n        # Temporarily disable metric computation, we will do it in the loop here.\n        compute_metrics = self.compute_metrics\n        self.compute_metrics = None\n        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop\n        start_time = time.time()\n        try:\n            output = eval_loop(\n                eval_dataloader,\n                description=\"Evaluation\",\n                # No point gathering the predictions if there are no metrics, otherwise we defer to\n                # self.args.prediction_loss_only\n                prediction_loss_only=True if compute_metrics is None else None,\n                ignore_keys=ignore_keys,\n                metric_key_prefix=metric_key_prefix,\n            )\n        finally:\n            self.compute_metrics = compute_metrics\n        total_batch_size = self.args.eval_batch_size * self.args.world_size\n        if f\"{metric_key_prefix}_jit_compilation_time\" in output.metrics:\n            start_time += output.metrics[f\"{metric_key_prefix}_jit_compilation_time\"]\n        output.metrics.update(\n            speed_metrics(\n                metric_key_prefix,\n                start_time,\n                num_samples=output.num_samples,\n                num_steps=math.ceil(output.num_samples / total_batch_size),\n            )\n        )\n        if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:\n            # Only the main node write the results by default\n            eval_preds = self.post_process_function(eval_examples, eval_dataset, output.predictions)\n            metrics = self.compute_metrics(eval_preds)\n\n            # Prefix all keys with metric_key_prefix + '_'\n            for key in list(metrics.keys()):\n                if not key.startswith(f\"{metric_key_prefix}_\"):\n                    metrics[f\"{metric_key_prefix}_{key}\"] = metrics.pop(key)\n            metrics.update(output.metrics)\n        else:\n            metrics = output.metrics\n\n        if self.args.should_log:\n            # Only the main node log the results by default\n            self.log(metrics)\n\n        if self.args.tpu_metrics_debug or self.args.debug:\n            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)\n            xm.master_print(met.metrics_report())\n\n        self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)\n        return metrics\n\n    def predict(self, predict_dataset, predict_examples, ignore_keys=None, metric_key_prefix: str = \"test\"):\n        predict_dataloader = self.get_test_dataloader(predict_dataset)\n\n        # Temporarily disable metric computation, we will do it in the loop here.\n        compute_metrics = self.compute_metrics\n        self.compute_metrics = None\n        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop\n        start_time = time.time()\n        try:\n            output = eval_loop(\n                predict_dataloader,\n                description=\"Prediction\",\n                # No point gathering the predictions if there are no metrics, otherwise we defer to\n                # self.args.prediction_loss_only\n                prediction_loss_only=True if compute_metrics is None else None,\n                ignore_keys=ignore_keys,\n                metric_key_prefix=metric_key_prefix,\n            )\n        finally:\n            self.compute_metrics = compute_metrics\n        total_batch_size = self.args.eval_batch_size * self.args.world_size\n        if f\"{metric_key_prefix}_jit_compilation_time\" in output.metrics:\n            start_time += output.metrics[f\"{metric_key_prefix}_jit_compilation_time\"]\n        output.metrics.update(\n            speed_metrics(\n                metric_key_prefix,\n                start_time,\n                num_samples=output.num_samples,\n                num_steps=math.ceil(output.num_samples / total_batch_size),\n            )\n        )\n\n        if self.post_process_function is None or self.compute_metrics is None:\n            return output\n\n        predictions = self.post_process_function(predict_examples, predict_dataset, output.predictions, \"predict\")\n        metrics = self.compute_metrics(predictions)\n\n        # Prefix all keys with metric_key_prefix + '_'\n        for key in list(metrics.keys()):\n            if not key.startswith(f\"{metric_key_prefix}_\"):\n                metrics[f\"{metric_key_prefix}_{key}\"] = metrics.pop(key)\n        metrics.update(output.metrics)\n\n\nSUMMARY:\n**Summary:**\n\nThe `QuestionAnsweringTrainer` class is a custom trainer for Question Answering Models, extending the `Trainer` class. Its primary purpose is to evaluate and make predictions on given datasets.\n\n**Main Purpose and Functionality:**\n\n* Evaluates `eval_dataset` using the `evaluate` method\n* Makes predictions on `predict_dataset` using the `predict` method\n* Applies post-processing function (`post_process_function`) to evaluation and prediction outputs\n\n**Key Implementation Details:**\n\n* `evaluate`: uses `prediction_loop` or `evaluation_loop`, computes metrics, and applies post-processing function\n* `predict`: uses `prediction_loop` or `evaluation_loop`, computes metrics, and applies post-processing function\n* Uses `compute_metrics` function to compute evaluation metrics\n\n**Notable Dependencies and Requirements:**\n\n* `Trainer` class (extended by `QuestionAnsweringTrainer`)\n* `post_process_function`\n* `eval_dataset` and `predict_dataset`\n* `compute_metrics` function\n* `prediction_loop` or `evaluation_loop`\n\n**Method Signatures and Interfaces:**\n\n* `evaluate(eval_dataset, eval_examples, ..., metric_key_prefix) -> metrics`\n* `predict(predict_dataset, predict_examples, ..., metric_key_prefix) -> output`"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa.py:227-705
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 705, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa.py", "name": "main", "start_line": 227, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_qa\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(\n            extension,\n            data_files=data_files,\n            field=\"data\",\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        use_fast=True,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForQuestionAnswering.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # Tokenizer check: this script requires a fast tokenizer.\n    if not isinstance(tokenizer, PreTrainedTokenizerFast):\n        raise ValueError(\n            \"This example script only works for models that have a fast tokenizer. Checkout the big table of models at\"\n            \" https://huggingface.co/transformers/index.html#supported-frameworks to find the model types that meet\"\n            \" this requirement\"\n        )\n\n    # Preprocessing the datasets.\n    # Preprocessing is slightly different for training and evaluation.\n    if training_args.do_train:\n        column_names = raw_datasets[\"train\"].column_names\n    elif training_args.do_eval:\n        column_names = raw_datasets[\"validation\"].column_names\n    else:\n        column_names = raw_datasets[\"test\"].column_names\n    question_column_name = \"question\" if \"question\" in column_names else column_names[0]\n    context_column_name = \"context\" if \"context\" in column_names else column_names[1]\n    answer_column_name = \"answers\" if \"answers\" in column_names else column_names[2]\n\n    # Padding side determines if we do (question|context) or (context|question).\n    pad_on_right = tokenizer.padding_side == \"right\"\n\n    if data_args.max_seq_length > tokenizer.model_max_length:\n        logger.warning(\n            f\"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the \"\n            f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n        )\n    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)\n\n    # Training preprocessing\n    def prepare_train_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=data_args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            padding=\"max_length\" if data_args.pad_to_max_length else False,\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n        # The offset mappings will give us a map from token to character position in the original context. This will\n        # help us compute the start_positions and end_positions.\n        offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n\n        # Let's label those examples!\n        tokenized_examples[\"start_positions\"] = []\n        tokenized_examples[\"end_positions\"] = []\n\n        for i, offsets in enumerate(offset_mapping):\n            # We will label impossible answers with the index of the CLS token.\n            input_ids = tokenized_examples[\"input_ids\"][i]\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples.sequence_ids(i)\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            answers = examples[answer_column_name][sample_index]\n            # If no answers are given, set the cls_index as answer.\n            if len(answers[\"answer_start\"]) == 0:\n                tokenized_examples[\"start_positions\"].append(cls_index)\n                tokenized_examples[\"end_positions\"].append(cls_index)\n            else:\n                # Start/end character index of the answer in the text.\n                start_char = answers[\"answer_start\"][0]\n                end_char = start_char + len(answers[\"text\"][0])\n\n                # Start token index of the current span in the text.\n                token_start_index = 0\n                while sequence_ids[token_start_index] != (1 if pad_on_right else 0):\n                    token_start_index += 1\n\n                # End token index of the current span in the text.\n                token_end_index = len(input_ids) - 1\n                while sequence_ids[token_end_index] != (1 if pad_on_right else 0):\n                    token_end_index -= 1\n\n                # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n                if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n                    tokenized_examples[\"start_positions\"].append(cls_index)\n                    tokenized_examples[\"end_positions\"].append(cls_index)\n                else:\n                    # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n                    # Note: we could go after the last offset if the answer is the last word (edge case).\n                    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n                        token_start_index += 1\n                    tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n                    while offsets[token_end_index][1] >= end_char:\n                        token_end_index -= 1\n                    tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n\n        return tokenized_examples\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            # We will select sample from whole data if argument is specified\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        # Create train feature from dataset\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                prepare_train_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n        if data_args.max_train_samples is not None:\n            # Number of samples might increase during Feature Creation, We select only specified max samples\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n\n    # Validation preprocessing\n    def prepare_validation_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=data_args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            padding=\"max_length\" if data_args.pad_to_max_length else False,\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n\n        # For evaluation, we will need to convert our predictions to substrings of the context, so we keep the\n        # corresponding example_id and we will store the offset mappings.\n        tokenized_examples[\"example_id\"] = []\n\n        for i in range(len(tokenized_examples[\"input_ids\"])):\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples.sequence_ids(i)\n            context_index = 1 if pad_on_right else 0\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n\n            # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n            # position is part of the context or not.\n            tokenized_examples[\"offset_mapping\"][i] = [\n                (o if sequence_ids[k] == context_index else None)\n                for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n            ]\n\n        return tokenized_examples\n\n    if training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_examples = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            # We will select sample from whole data\n            max_eval_samples = min(len(eval_examples), data_args.max_eval_samples)\n            eval_examples = eval_examples.select(range(max_eval_samples))\n        # Validation Feature Creation\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n        if data_args.max_eval_samples is not None:\n            # During Feature creation dataset samples might increase, we will select required samples again\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n\n    if training_args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_examples = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            # We will select sample from whole data\n            predict_examples = predict_examples.select(range(data_args.max_predict_samples))\n        # Predict Feature Creation\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n        if data_args.max_predict_samples is not None:\n            # During Feature creation dataset samples might increase, we will select required samples again\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n\n    # Data collator\n    # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data\n    # collator.\n    data_collator = (\n        default_data_collator\n        if data_args.pad_to_max_length\n        else DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None)\n    )\n\n    # Post-processing:\n    def post_processing_function(examples, features, predictions, stage=\"eval\"):\n        # Post-processing: we match the start logits and end logits to answers in the original context.\n        predictions = postprocess_qa_predictions(\n            examples=examples,\n            features=features,\n            predictions=predictions,\n            version_2_with_negative=data_args.version_2_with_negative,\n            n_best_size=data_args.n_best_size,\n            max_answer_length=data_args.max_answer_length,\n            null_score_diff_threshold=data_args.null_score_diff_threshold,\n            output_dir=training_args.output_dir,\n            log_level=log_level,\n            prefix=stage,\n        )\n        # Format the result to the format the metric expects.\n        if data_args.version_2_with_negative:\n            formatted_predictions = [\n                {\"id\": str(k), \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in predictions.items()\n            ]\n        else:\n            formatted_predictions = [{\"id\": str(k), \"prediction_text\": v} for k, v in predictions.items()]\n\n        references = [{\"id\": str(ex[\"id\"]), \"answers\": ex[answer_column_name]} for ex in examples]\n        return EvalPrediction(predictions=formatted_predictions, label_ids=references)\n\n    if data_args.version_2_with_negative:\n        accepted_best_metrics = (\"exact\", \"f1\", \"HasAns_exact\", \"HasAns_f1\")\n    else:\n        accepted_best_metrics = (\"exact_match\", \"f1\")\n\n    if training_args.load_best_model_at_end and training_args.metric_for_best_model not in accepted_best_metrics:\n        warnings.warn(f\"--metric_for_best_model should be set to one of {accepted_best_metrics}\")\n\n    metric = evaluate.load(\n        \"squad_v2\" if data_args.version_2_with_negative else \"squad\", cache_dir=model_args.cache_dir\n    )\n\n    def compute_metrics(p: EvalPrediction):\n        return metric.compute(predictions=p.predictions, references=p.label_ids)\n\n    # Initialize our Trainer\n    trainer = QuestionAnsweringTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        eval_examples=eval_examples if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        post_process_function=post_processing_function,\n        compute_metrics=compute_metrics,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        metrics = train_result.metrics\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n        metrics = trainer.evaluate()\n\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Prediction\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n        results = trainer.predict(predict_dataset, predict_examples)\n        metrics = results.metrics\n\n        max_predict_samples = (\n            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n        )\n        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n    kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"question-answering\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nHere is a concise summary of the code:\n\n**Overview**: This is a script for training and evaluating a Question Answering (QA) model using the Hugging Face Transformers library. The purpose is to fine-tune a pre-trained QA model on a specific dataset.\n\n**Key Components**:\n\n1. **Model Training**: The script loads a pre-trained QA model, sets hyperparameters, and trains the model on a training dataset.\n2. **Model Evaluation**: After training, the script evaluates the model on an evaluation dataset using metrics such as exact match, F1 score, etc.\n3. **Model Prediction**: If required, the script can predict answers for a new test dataset.\n\n**Configuration Options**:\n\n* `data_args`: A dictionary of configuration options, including dataset name, version number (v2 or v2_with_negative), and hyperparameters.\n* `training_args`: A dictionary of training arguments, such as resume from checkpoint, save model, etc.\n\n**Output**:\n\n* The script saves the trained model to a specific directory.\n* It also logs metrics during training and evaluation phases.\n* If required, it can predict answers for new test datasets."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa.py:708-710
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 710, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa.py", "name": "_mp_fn", "start_line": 708, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Summary:**\n\n`_mp_fn(index)` is a recursive function that executes the `main()` function when working with XLA Spawn on TPU's. Specifically, it serves as a worker function for multi-processing and parallel computing.\n\n**Key Details:**\n\n* Primary purpose: Execute `main()` in a multi-processing context.\n* Algorithm/pattern: Recursive function call to utilize multiple processes/cores.\n\n**Notable Dependencies/R requirements:**\n\n* None explicitly listed, but implies:\n  + `xla_spawn`: a library for XLA (Accelerated Linear Algebra) and TPU's.\n  + Python multiprocessing capabilities."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py:69-89

metadata:
{"docstring": "", "end_line": 89, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py", "name": "save_prefixed_metrics", "start_line": 69, "type": "function", "chroma:document": "CODE:\n\ndef save_prefixed_metrics(results, output_dir, file_name: str = \"all_results.json\", metric_key_prefix: str = \"eval\"):\n    \"\"\"\n    Save results while prefixing metric names.\n\n    Args:\n        results: (:obj:`dict`):\n            A dictionary of results.\n        output_dir: (:obj:`str`):\n            An output directory.\n        file_name: (:obj:`str`, `optional`, defaults to :obj:`all_results.json`):\n            An output file name.\n        metric_key_prefix: (:obj:`str`, `optional`, defaults to :obj:`eval`):\n            A metric name prefix.\n    \"\"\"\n    # Prefix all keys with metric_key_prefix + '_'\n    for key in list(results.keys()):\n        if not key.startswith(f\"{metric_key_prefix}_\"):\n            results[f\"{metric_key_prefix}_{key}\"] = results.pop(key)\n\n    with open(os.path.join(output_dir, file_name), \"w\") as f:\n\n\nSUMMARY:\n**Function Summary: `save_prefixed_metrics`**\n\n### Purpose and Functionality\n\nThe `save_prefixed_metrics` function saves results to a JSON file while prefixing metric names. It takes in a dictionary of results, an output directory, and optional parameters for file name and metric key prefix.\n\n### Key Implementation Details\n\n* Prefixes all keys with the specified `metric_key_prefix`.\n* Utilizes `json.dump()` to write the results to a file.\n\n### Notable Dependencies or Requirements\n\n* Uses `os` module for path manipulation.\n* Relying on `json` module for data serialization."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py:92-335

metadata:
{"docstring": "", "end_line": 335, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py", "name": "parse_args", "start_line": 92, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a Question Answering task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\", type=int, default=1, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\"--do_predict\", action=\"store_true\", help=\"To do prediction on the question answering model\")\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--test_file\", type=str, default=None, help=\"A csv or a json file containing the Prediction data.\"\n    )\n    parser.add_argument(\n        \"--max_seq_length\",\n        type=int,\n        default=384,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,\"\n            \" sequences shorter will be padded if `--pad_to_max_length` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_seq_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--doc_stride\",\n        type=int,\n        default=128,\n        help=\"When splitting up a long document into chunks how much stride to take between chunks.\",\n    )\n    parser.add_argument(\n        \"--n_best_size\",\n        type=int,\n        default=20,\n        help=\"The total number of n-best predictions to generate when looking for an answer.\",\n    )\n    parser.add_argument(\n        \"--null_score_diff_threshold\",\n        type=float,\n        default=0.0,\n        help=(\n            \"The threshold used to select the null answer: if the best answer has a score that is less than \"\n            \"the score of the null answer minus this threshold, the null answer is selected for this example. \"\n            \"Only useful when `version_2_with_negative=True`.\"\n        ),\n    )\n    parser.add_argument(\n        \"--version_2_with_negative\",\n        action=\"store_true\",\n        help=\"If true, some of the examples do not have an answer.\",\n    )\n    parser.add_argument(\n        \"--max_answer_length\",\n        type=int,\n        default=30,\n        help=(\n            \"The maximum length of an answer that can be generated. This is needed because the start \"\n            \"and end predictions are not conditioned on one another.\"\n        ),\n    )\n    parser.add_argument(\n        \"--max_train_samples\",\n        type=int,\n        default=None,\n        help=(\n            \"For debugging purposes or quicker training, truncate the number of training examples to this \"\n            \"value if set.\"\n        ),\n    )\n    parser.add_argument(\n        \"--max_eval_samples\",\n        type=int,\n        default=None,\n        help=(\n            \"For debugging purposes or quicker training, truncate the number of evaluation examples to this \"\n            \"value if set.\"\n        ),\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--max_predict_samples\",\n        type=int,\n        default=None,\n        help=\"For debugging purposes or quicker training, truncate the number of prediction examples to this\",\n    )\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if (\n        args.dataset_name is None\n        and args.train_file is None\n        and args.validation_file is None\n        and args.test_file is None\n    ):\n        raise ValueError(\"Need either a dataset name or a training/validation/test file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n        if args.test_file is not None:\n            extension = args.test_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`test_file` should be a csv or a json file.\"\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\nThis function defines a script for fine-tuning a transformer model on a Question Answering (QA) task. The script takes in various command-line arguments that configure the training process, such as:\n\n* Data files and locations\n* Model type and parameters\n* Training and validation settings\n* Overwrite cache and push to hub options\n\nThe function performs sanity checks to ensure consistency and validity of input arguments. It then parses the argument using `argparse` and returns a `Namespace` object containing all the arguments.\n\nIn essence, this script allows users to customize the training process for a transformer-based QA model and provides a flexible way to perform fine-tuning with various configuration options."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py:338-1039

metadata:
{"docstring": "", "end_line": 1039, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_no_trainer.py", "name": "main", "start_line": 338, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_qa_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        if args.test_file is not None:\n            data_files[\"test\"] = args.test_file\n            extension = args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files, field=\"data\")\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n\n    if args.config_name:\n        config = AutoConfig.from_pretrained(args.config_name, trust_remote_code=args.trust_remote_code)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=True, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=True, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForQuestionAnswering.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForQuestionAnswering.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # Preprocessing the datasets.\n    # Preprocessing is slightly different for training and evaluation.\n\n    column_names = raw_datasets[\"train\"].column_names\n\n    question_column_name = \"question\" if \"question\" in column_names else column_names[0]\n    context_column_name = \"context\" if \"context\" in column_names else column_names[1]\n    answer_column_name = \"answers\" if \"answers\" in column_names else column_names[2]\n\n    # Padding side determines if we do (question|context) or (context|question).\n    pad_on_right = tokenizer.padding_side == \"right\"\n\n    if args.max_seq_length > tokenizer.model_max_length:\n        logger.warning(\n            f\"The max_seq_length passed ({args.max_seq_length}) is larger than the maximum length for the \"\n            f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n        )\n\n    max_seq_length = min(args.max_seq_length, tokenizer.model_max_length)\n\n    # Training preprocessing\n    def prepare_train_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            padding=\"max_length\" if args.pad_to_max_length else False,\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n        # The offset mappings will give us a map from token to character position in the original context. This will\n        # help us compute the start_positions and end_positions.\n        offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n\n        # Let's label those examples!\n        tokenized_examples[\"start_positions\"] = []\n        tokenized_examples[\"end_positions\"] = []\n\n        for i, offsets in enumerate(offset_mapping):\n            # We will label impossible answers with the index of the CLS token.\n            input_ids = tokenized_examples[\"input_ids\"][i]\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples.sequence_ids(i)\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            answers = examples[answer_column_name][sample_index]\n            # If no answers are given, set the cls_index as answer.\n            if len(answers[\"answer_start\"]) == 0:\n                tokenized_examples[\"start_positions\"].append(cls_index)\n                tokenized_examples[\"end_positions\"].append(cls_index)\n            else:\n                # Start/end character index of the answer in the text.\n                start_char = answers[\"answer_start\"][0]\n                end_char = start_char + len(answers[\"text\"][0])\n\n                # Start token index of the current span in the text.\n                token_start_index = 0\n                while sequence_ids[token_start_index] != (1 if pad_on_right else 0):\n                    token_start_index += 1\n\n                # End token index of the current span in the text.\n                token_end_index = len(input_ids) - 1\n                while sequence_ids[token_end_index] != (1 if pad_on_right else 0):\n                    token_end_index -= 1\n\n                # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n                if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n                    tokenized_examples[\"start_positions\"].append(cls_index)\n                    tokenized_examples[\"end_positions\"].append(cls_index)\n                else:\n                    # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n                    # Note: we could go after the last offset if the answer is the last word (edge case).\n                    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n                        token_start_index += 1\n                    tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n                    while offsets[token_end_index][1] >= end_char:\n                        token_end_index -= 1\n                    tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n\n        return tokenized_examples\n\n    if \"train\" not in raw_datasets:\n        raise ValueError(\"--do_train requires a train dataset\")\n    train_dataset = raw_datasets[\"train\"]\n    if args.max_train_samples is not None:\n        # We will select sample from whole data if argument is specified\n        train_dataset = train_dataset.select(range(args.max_train_samples))\n\n    # Create train feature from dataset\n    with accelerator.main_process_first():\n        train_dataset = train_dataset.map(\n            prepare_train_features,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on train dataset\",\n        )\n        if args.max_train_samples is not None:\n            # Number of samples might increase during Feature Creation, We select only specified max samples\n            train_dataset = train_dataset.select(range(args.max_train_samples))\n\n    # Validation preprocessing\n    def prepare_validation_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            padding=\"max_length\" if args.pad_to_max_length else False,\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n\n        # For evaluation, we will need to convert our predictions to substrings of the context, so we keep the\n        # corresponding example_id and we will store the offset mappings.\n        tokenized_examples[\"example_id\"] = []\n\n        for i in range(len(tokenized_examples[\"input_ids\"])):\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples.sequence_ids(i)\n            context_index = 1 if pad_on_right else 0\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n\n            # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n            # position is part of the context or not.\n            tokenized_examples[\"offset_mapping\"][i] = [\n                (o if sequence_ids[k] == context_index else None)\n                for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n            ]\n\n        return tokenized_examples\n\n    if \"validation\" not in raw_datasets:\n        raise ValueError(\"--do_eval requires a validation dataset\")\n    eval_examples = raw_datasets[\"validation\"]\n    if args.max_eval_samples is not None:\n        # We will select sample from whole data\n        eval_examples = eval_examples.select(range(args.max_eval_samples))\n    # Validation Feature Creation\n    with accelerator.main_process_first():\n        eval_dataset = eval_examples.map(\n            prepare_validation_features,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on validation dataset\",\n        )\n\n    if args.max_eval_samples is not None:\n        # During Feature creation dataset samples might increase, we will select required samples again\n        eval_dataset = eval_dataset.select(range(args.max_eval_samples))\n\n    if args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_examples = raw_datasets[\"test\"]\n        if args.max_predict_samples is not None:\n            # We will select sample from whole data\n            predict_examples = predict_examples.select(range(args.max_predict_samples))\n        # Predict Feature Creation\n        with accelerator.main_process_first():\n            predict_dataset = predict_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n            if args.max_predict_samples is not None:\n                # During Feature creation dataset samples might increase, we will select required samples again\n                predict_dataset = predict_dataset.select(range(args.max_predict_samples))\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    if args.pad_to_max_length:\n        # If padding was already done ot max length, we use the default data collator that will just convert everything\n        # to tensors.\n        data_collator = default_data_collator\n    else:\n        # Otherwise, `DataCollatorWithPadding` will apply dynamic padding for us (by padding to the maximum length of\n        # the samples passed). When using mixed precision, we add `pad_to_multiple_of=8` to pad all tensors to multiple\n        # of 8s, which will enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\n        # For fp8, we pad to multiple of 16.\n        if accelerator.mixed_precision == \"fp8\":\n            pad_to_multiple_of = 16\n        elif accelerator.mixed_precision != \"no\":\n            pad_to_multiple_of = 8\n        else:\n            pad_to_multiple_of = None\n        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=pad_to_multiple_of)\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n\n    eval_dataset_for_model = eval_dataset.remove_columns([\"example_id\", \"offset_mapping\"])\n    eval_dataloader = DataLoader(\n        eval_dataset_for_model, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size\n    )\n\n    if args.do_predict:\n        predict_dataset_for_model = predict_dataset.remove_columns([\"example_id\", \"offset_mapping\"])\n        predict_dataloader = DataLoader(\n            predict_dataset_for_model, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size\n        )\n\n    # Post-processing:\n    def post_processing_function(examples, features, predictions, stage=\"eval\"):\n        # Post-processing: we match the start logits and end logits to answers in the original context.\n        predictions = postprocess_qa_predictions(\n            examples=examples,\n            features=features,\n            predictions=predictions,\n            version_2_with_negative=args.version_2_with_negative,\n            n_best_size=args.n_best_size,\n            max_answer_length=args.max_answer_length,\n            null_score_diff_threshold=args.null_score_diff_threshold,\n            output_dir=args.output_dir,\n            prefix=stage,\n        )\n        # Format the result to the format the metric expects.\n        if args.version_2_with_negative:\n            formatted_predictions = [\n                {\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in predictions.items()\n            ]\n        else:\n            formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in predictions.items()]\n\n        references = [{\"id\": ex[\"id\"], \"answers\": ex[answer_column_name]} for ex in examples]\n        return EvalPrediction(predictions=formatted_predictions, label_ids=references)\n\n    metric = evaluate.load(\"squad_v2\" if args.version_2_with_negative else \"squad\")\n\n    # Create and fill numpy array of size len_of_validation_data * max_length_of_output_tensor\n    def create_and_fill_np_array(start_or_end_logits, dataset, max_len):\n        \"\"\"\n        Create and fill numpy array of size len_of_validation_data * max_length_of_output_tensor\n\n        Args:\n            start_or_end_logits(:obj:`tensor`):\n                This is the output predictions of the model. We can only enter either start or end logits.\n            eval_dataset: Evaluation dataset\n            max_len(:obj:`int`):\n                The maximum length of the output tensor. ( See the model.eval() part for more details )\n        \"\"\"\n\n        step = 0\n        # create a numpy array and fill it with -100.\n        logits_concat = np.full((len(dataset), max_len), -100, dtype=np.float64)\n        # Now since we have create an array now we will populate it with the outputs gathered using accelerator.gather_for_metrics\n        for i, output_logit in enumerate(start_or_end_logits):  # populate columns\n            # We have to fill it such that we have to take the whole tensor and replace it on the newly created array\n            # And after every iteration we have to change the step\n\n            batch_size = output_logit.shape[0]\n            cols = output_logit.shape[1]\n\n            if step + batch_size < len(dataset):\n                logits_concat[step : step + batch_size, :cols] = output_logit\n            else:\n                logits_concat[step:, :cols] = output_logit[: len(dataset) - step]\n\n            step += batch_size\n\n        return logits_concat\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"qa_no_trainer\", experiment_config)\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n    # Evaluation\n    logger.info(\"***** Running Evaluation *****\")\n    logger.info(f\"  Num examples = {len(eval_dataset)}\")\n    logger.info(f\"  Batch size = {args.per_device_eval_batch_size}\")\n\n    all_start_logits = []\n    all_end_logits = []\n\n    model.eval()\n\n    for step, batch in enumerate(eval_dataloader):\n        with torch.no_grad():\n            outputs = model(**batch)\n            start_logits = outputs.start_logits\n            end_logits = outputs.end_logits\n\n            if not args.pad_to_max_length:  # necessary to pad predictions and labels for being gathered\n                start_logits = accelerator.pad_across_processes(start_logits, dim=1, pad_index=-100)\n                end_logits = accelerator.pad_across_processes(end_logits, dim=1, pad_index=-100)\n\n            all_start_logits.append(accelerator.gather_for_metrics(start_logits).cpu().numpy())\n            all_end_logits.append(accelerator.gather_for_metrics(end_logits).cpu().numpy())\n\n    max_len = max([x.shape[1] for x in all_start_logits])  # Get the max_length of the tensor\n\n    # concatenate the numpy array\n    start_logits_concat = create_and_fill_np_array(all_start_logits, eval_dataset, max_len)\n    end_logits_concat = create_and_fill_np_array(all_end_logits, eval_dataset, max_len)\n\n    # delete the list of numpy arrays\n    del all_start_logits\n    del all_end_logits\n\n    outputs_numpy = (start_logits_concat, end_logits_concat)\n    prediction = post_processing_function(eval_examples, eval_dataset, outputs_numpy)\n    eval_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)\n    logger.info(f\"Evaluation metrics: {eval_metric}\")\n\n    # Prediction\n    if args.do_predict:\n        logger.info(\"***** Running Prediction *****\")\n        logger.info(f\"  Num examples = {len(predict_dataset)}\")\n        logger.info(f\"  Batch size = {args.per_device_eval_batch_size}\")\n\n        all_start_logits = []\n        all_end_logits = []\n\n        model.eval()\n\n        for step, batch in enumerate(predict_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n                start_logits = outputs.start_logits\n                end_logits = outputs.end_logits\n\n                if not args.pad_to_max_length:  # necessary to pad predictions and labels for being gathered\n                    start_logits = accelerator.pad_across_processes(start_logits, dim=1, pad_index=-100)\n                    end_logits = accelerator.pad_across_processes(end_logits, dim=1, pad_index=-100)\n\n                all_start_logits.append(accelerator.gather_for_metrics(start_logits).cpu().numpy())\n                all_end_logits.append(accelerator.gather_for_metrics(end_logits).cpu().numpy())\n\n        max_len = max([x.shape[1] for x in all_start_logits])  # Get the max_length of the tensor\n        # concatenate the numpy array\n        start_logits_concat = create_and_fill_np_array(all_start_logits, predict_dataset, max_len)\n        end_logits_concat = create_and_fill_np_array(all_end_logits, predict_dataset, max_len)\n\n        # delete the list of numpy arrays\n        del all_start_logits\n        del all_end_logits\n\n        outputs_numpy = (start_logits_concat, end_logits_concat)\n        prediction = post_processing_function(predict_examples, predict_dataset, outputs_numpy)\n        predict_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)\n        logger.info(f\"Predict metrics: {predict_metric}\")\n\n    if args.with_tracking:\n        log = {\n            \"squad_v2\" if args.version_2_with_negative else \"squad\": eval_metric,\n            \"train_loss\": total_loss.item() / len(train_dataloader),\n            \"epoch\": epoch,\n            \"step\": completed_steps,\n        }\n    if args.do_predict:\n        log[\"squad_v2_predict\" if args.version_2_with_negative else \"squad_predict\"] = predict_metric\n\n        accelerator.log(log, step=completed_steps)\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n            logger.info(json.dumps(eval_metric, indent=4))\n\n\nSUMMARY:\nThis is a PyTorch script for training and evaluating a sequence-to-sequence model using the SQuADv2 dataset. The script covers various aspects of model training, including:\n\n1. Setting up the training environment\n2. Defining the model architecture and hyperparameters\n3. Training the model on the SQuADv2 dataset\n4. Evaluating the model's performance using precision and recall metrics\n5. Logging progress and results\n6. Saving the trained model to a local directory\n\nThe script is well-structured, with clear section headings and concise comments. However, it is quite long and might be improved with additional documentation or annotations.\n\nSome minor issues and suggestions for improvement are:\n\n1. The `create_and_fill_np_array` function is not defined within this script.\n2. The `post_processing_function` and `metric` functions are not defined here either.\n3. The `save_prefixed_metrics` function is called without being imported from any module.\n\nTo make the code more concise, some suggestions are:\n\n1. Extract common functions into separate modules to improve readability and maintainability.\n2. Use type hints for the script's variables and function parameters to enhance clarity.\n3. Consider using logging libraries (e.g., `logging`) instead of custom logging statements to conform with standard Python practices.\n\nHere's a minor refactored version incorporating some of these suggestions:\n```python\nimport torch\nfrom typing import Dict\n\n# Define post-processing and evaluation functions (e.g., in separate modules)\n\ndef create_and_fill_np_array(data, max_len):\n    # Function definition remains the same\n    pass\n\ndef compute_metrics(predictions, labels):\n    # Implementation omitted for brevity\n    return predictions, labels\n\ndef save_prefixed_metrics(eval_metric, output_dir):\n    # Import necessary module (e.g., `logging`) and implementation adjusted accordingly\n    import logging\n    logging.info(\"Evaluation metrics: \" + str(eval_metric))\n    # ...\n\n# Training script remains largely unchanged...\n\nif args.with_tracking:\n    log = {\n        \"squad_v2\" if args.version_2_with_negative else \"squad\": eval_metric,\n        \"train_loss\": total_loss.item() / len(train_dataloader),\n        \"epoch\": epoch,\n        \"step\": completed_steps,\n    }\nif args.do_predict:\n    # ...\n\n# Training the model\noptimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)\nfor epoch in range(args.num_epochs):\n    # ...\n```\nPlease adjust the refactoring according to your specific needs and requirements."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search.py:225-732

metadata:
{"docstring": "", "end_line": 732, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search.py", "name": "main", "start_line": 225, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_qa_beam_search\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=data_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(\n            extension,\n            data_files=data_files,\n            field=\"data\",\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = XLNetConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n    )\n    tokenizer = XLNetTokenizerFast.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n    )\n    model = XLNetForQuestionAnswering.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n    )\n\n    # Preprocessing the datasets.\n    # Preprocessing is slightly different for training and evaluation.\n    if training_args.do_train:\n        column_names = raw_datasets[\"train\"].column_names\n    elif training_args.do_eval:\n        column_names = raw_datasets[\"validation\"].column_names\n    else:\n        column_names = raw_datasets[\"test\"].column_names\n    question_column_name = \"question\" if \"question\" in column_names else column_names[0]\n    context_column_name = \"context\" if \"context\" in column_names else column_names[1]\n    answer_column_name = \"answers\" if \"answers\" in column_names else column_names[2]\n\n    # Padding side determines if we do (question|context) or (context|question).\n    pad_on_right = tokenizer.padding_side == \"right\"\n\n    if data_args.max_seq_length > tokenizer.model_max_length:\n        logger.warning(\n            f\"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the \"\n            f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n        )\n    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)\n\n    # Training preprocessing\n    def prepare_train_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=data_args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            return_special_tokens_mask=True,\n            return_token_type_ids=True,\n            padding=\"max_length\",\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n        # The offset mappings will give us a map from token to character position in the original context. This will\n        # help us compute the start_positions and end_positions.\n        offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n        # The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).\n        special_tokens = tokenized_examples.pop(\"special_tokens_mask\")\n\n        # Let's label those examples!\n        tokenized_examples[\"start_positions\"] = []\n        tokenized_examples[\"end_positions\"] = []\n        tokenized_examples[\"is_impossible\"] = []\n        tokenized_examples[\"cls_index\"] = []\n        tokenized_examples[\"p_mask\"] = []\n\n        for i, offsets in enumerate(offset_mapping):\n            # We will label impossible answers with the index of the CLS token.\n            input_ids = tokenized_examples[\"input_ids\"][i]\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n            tokenized_examples[\"cls_index\"].append(cls_index)\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples[\"token_type_ids\"][i]\n            for k, s in enumerate(special_tokens[i]):\n                if s:\n                    sequence_ids[k] = 3\n            context_idx = 1 if pad_on_right else 0\n\n            # Build the p_mask: non special tokens and context gets 0.0, the others get 1.0.\n            # The cls token gets 1.0 too (for predictions of empty answers).\n            tokenized_examples[\"p_mask\"].append(\n                [\n                    0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0\n                    for k, s in enumerate(sequence_ids)\n                ]\n            )\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            answers = examples[answer_column_name][sample_index]\n            # If no answers are given, set the cls_index as answer.\n            if len(answers[\"answer_start\"]) == 0:\n                tokenized_examples[\"start_positions\"].append(cls_index)\n                tokenized_examples[\"end_positions\"].append(cls_index)\n                tokenized_examples[\"is_impossible\"].append(1.0)\n            else:\n                # Start/end character index of the answer in the text.\n                start_char = answers[\"answer_start\"][0]\n                end_char = start_char + len(answers[\"text\"][0])\n\n                # Start token index of the current span in the text.\n                token_start_index = 0\n                while sequence_ids[token_start_index] != context_idx:\n                    token_start_index += 1\n\n                # End token index of the current span in the text.\n                token_end_index = len(input_ids) - 1\n                while sequence_ids[token_end_index] != context_idx:\n                    token_end_index -= 1\n                # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n                if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n                    tokenized_examples[\"start_positions\"].append(cls_index)\n                    tokenized_examples[\"end_positions\"].append(cls_index)\n                    tokenized_examples[\"is_impossible\"].append(1.0)\n                else:\n                    # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n                    # Note: we could go after the last offset if the answer is the last word (edge case).\n                    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n                        token_start_index += 1\n                    tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n                    while offsets[token_end_index][1] >= end_char:\n                        token_end_index -= 1\n                    tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n                    tokenized_examples[\"is_impossible\"].append(0.0)\n\n        return tokenized_examples\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            # Select samples from Dataset, This will help to decrease processing time\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        # Create Training Features\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                prepare_train_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n        if data_args.max_train_samples is not None:\n            # Select samples from dataset again since Feature Creation might increase number of features\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n\n    # Validation preprocessing\n    def prepare_validation_features(examples):\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=data_args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            return_special_tokens_mask=True,\n            return_token_type_ids=True,\n            padding=\"max_length\",\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n\n        # The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).\n        special_tokens = tokenized_examples.pop(\"special_tokens_mask\")\n\n        # For evaluation, we will need to convert our predictions to substrings of the context, so we keep the\n        # corresponding example_id and we will store the offset mappings.\n        tokenized_examples[\"example_id\"] = []\n\n        # We still provide the index of the CLS token and the p_mask to the model, but not the is_impossible label.\n        tokenized_examples[\"cls_index\"] = []\n        tokenized_examples[\"p_mask\"] = []\n\n        for i, input_ids in enumerate(tokenized_examples[\"input_ids\"]):\n            # Find the CLS token in the input ids.\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n            tokenized_examples[\"cls_index\"].append(cls_index)\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples[\"token_type_ids\"][i]\n            for k, s in enumerate(special_tokens[i]):\n                if s:\n                    sequence_ids[k] = 3\n            context_idx = 1 if pad_on_right else 0\n\n            # Build the p_mask: non special tokens and context gets 0.0, the others 1.0.\n            tokenized_examples[\"p_mask\"].append(\n                [\n                    0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0\n                    for k, s in enumerate(sequence_ids)\n                ]\n            )\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n\n            # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n            # position is part of the context or not.\n            tokenized_examples[\"offset_mapping\"][i] = [\n                (o if sequence_ids[k] == context_idx else None)\n                for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n            ]\n\n        return tokenized_examples\n\n    if training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_examples = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            # Selecting Eval Samples from Dataset\n            max_eval_samples = min(len(eval_examples), data_args.max_eval_samples)\n            eval_examples = eval_examples.select(range(max_eval_samples))\n        # Create Features from Eval Dataset\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n        if data_args.max_eval_samples is not None:\n            # Selecting Samples from Dataset again since Feature Creation might increase samples size\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n\n    if training_args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_examples = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            # We will select sample from whole data\n            predict_examples = predict_examples.select(range(data_args.max_predict_samples))\n        # Test Feature Creation\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n        if data_args.max_predict_samples is not None:\n            # During Feature creation dataset samples might increase, we will select required samples again\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n\n    # Data collator\n    # We have already padded to max length if the corresponding flag is True, otherwise we need to pad in the data\n    # collator.\n    data_collator = (\n        default_data_collator\n        if data_args.pad_to_max_length\n        else DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None)\n    )\n\n    # Post-processing:\n    def post_processing_function(examples, features, predictions, stage=\"eval\"):\n        # Post-processing: we match the start logits and end logits to answers in the original context.\n        predictions, scores_diff_json = postprocess_qa_predictions_with_beam_search(\n            examples=examples,\n            features=features,\n            predictions=predictions,\n            version_2_with_negative=data_args.version_2_with_negative,\n            n_best_size=data_args.n_best_size,\n            max_answer_length=data_args.max_answer_length,\n            start_n_top=model.config.start_n_top,\n            end_n_top=model.config.end_n_top,\n            output_dir=training_args.output_dir,\n            log_level=log_level,\n            prefix=stage,\n        )\n        # Format the result to the format the metric expects.\n        if data_args.version_2_with_negative:\n            formatted_predictions = [\n                {\"id\": k, \"prediction_text\": v, \"no_answer_probability\": scores_diff_json[k]}\n                for k, v in predictions.items()\n            ]\n        else:\n            formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in predictions.items()]\n\n        references = [{\"id\": ex[\"id\"], \"answers\": ex[answer_column_name]} for ex in examples]\n        return EvalPrediction(predictions=formatted_predictions, label_ids=references)\n\n    metric = evaluate.load(\n        \"squad_v2\" if data_args.version_2_with_negative else \"squad\", cache_dir=model_args.cache_dir\n    )\n\n    def compute_metrics(p: EvalPrediction):\n        return metric.compute(predictions=p.predictions, references=p.label_ids)\n\n    # Initialize our Trainer\n    trainer = QuestionAnsweringTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        eval_examples=eval_examples if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        post_process_function=post_processing_function,\n        compute_metrics=compute_metrics,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        metrics = train_result.metrics\n\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n        metrics = trainer.evaluate()\n\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Prediction\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n        results = trainer.predict(predict_dataset, predict_examples)\n        metrics = results.metrics\n\n        max_predict_samples = (\n            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n        )\n        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n    kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"question-answering\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nThis is an example Python script that trains a Question Answering (QA) model using the Hugging Face Transformers library. The script uses a pre-trained QA model and fine-tunes it on a specific dataset.\n\nHere's a brief summary of the code:\n\n1. **Data preparation**: The script loads the trained pre-trained QA model, tokenizer, and dataset.\n2. **Training setup**: The script sets up the training process by creating an instance of the `QuestionAnsweringTrainer` class, which takes the model, arguments, train dataset, evaluation datasets, and other configuration options as inputs.\n3. **Training loop**: The script trains the model using the `train` method of the `Trainer` instance, which calls the `train` function with various parameters such as batch size, number of epochs, and optimization algorithm.\n4. **Evaluation and prediction**: After training, the script evaluates the model on a hold-out test dataset and generates predictions for new, unseen data.\n\nThe script is likely part of a larger project that aims to train high-performance models using efficient algorithms and computational resources."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search.py:735-737

metadata:
{"docstring": "", "end_line": 737, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search.py", "name": "_mp_fn", "start_line": 735, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Function Summary**\n\n```markdown\n# Function Name: `_mp_fn(index)`\n# Purpose: Main entry point for multiprocessing on TPUs using XLA Spawn.\n\n# Key Details:\n# - Calls the `main()` function as the primary execution target under XLA Spawn.\n```\n\nNote: This code chunk does not provide sufficient information to determine the purpose of the `main()` call, which makes it difficult to generate a comprehensive summary. If more context is available, I can attempt to add more details."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py:65-85

metadata:
{"docstring": "", "end_line": 85, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py", "name": "save_prefixed_metrics", "start_line": 65, "type": "function", "chroma:document": "CODE:\n\ndef save_prefixed_metrics(results, output_dir, file_name: str = \"all_results.json\", metric_key_prefix: str = \"eval\"):\n    \"\"\"\n    Save results while prefixing metric names.\n\n    Args:\n        results: (:obj:`dict`):\n            A dictionary of results.\n        output_dir: (:obj:`str`):\n            An output directory.\n        file_name: (:obj:`str`, `optional`, defaults to :obj:`all_results.json`):\n            An output file name.\n        metric_key_prefix: (:obj:`str`, `optional`, defaults to :obj:`eval`):\n            A metric name prefix.\n    \"\"\"\n    # Prefix all keys with metric_key_prefix + '_'\n    for key in list(results.keys()):\n        if not key.startswith(f\"{metric_key_prefix}_\"):\n            results[f\"{metric_key_prefix}_{key}\"] = results.pop(key)\n\n    with open(os.path.join(output_dir, file_name), \"w\") as f:\n\n\nSUMMARY:\n**Summary:**\n\n### save_prefixed_metrics Function\n\n*   Saves prefixed metric results to a JSON file.\n*   Modifies input dictionary `results` by prefixing keys with a specified name (`metric_key_prefix`) if necessary.\n\n#### Key Details:\n\n*   **Function Argument:** \n    ```python\ndef save_prefixed_metrics(results: dict, output_dir: str, file_name: str = \"all_results.json\", metric_key_prefix: str = \"eval\")\n```\n*   **Data Structure Manipulation:** Prefixes dictionary keys with `metric_key_prefix` using `dict.pop()` and f-strings for renaming.\n*   **File Handling:** Saves modified dictionary to a JSON file with specified `output_dir` and `file_name`.\n\n#### Dependencies:\n\n*   External dependencies: \n    *   `os`: module used for interacting with the operating system, e.g., joining path components together (`os.path.join`)."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py:88-297

metadata:
{"docstring": "", "end_line": 297, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py", "name": "parse_args", "start_line": 88, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a Question Answering task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\", type=int, default=1, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\"--do_predict\", action=\"store_true\", help=\"Eval the question answering model\")\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--test_file\", type=str, default=None, help=\"A csv or a json file containing the Prediction data.\"\n    )\n    parser.add_argument(\n        \"--max_seq_length\",\n        type=int,\n        default=384,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,\"\n            \" sequences shorter will be padded if `--pad_to_max_length` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_seq_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=True,\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--doc_stride\",\n        type=int,\n        default=128,\n        help=\"When splitting up a long document into chunks how much stride to take between chunks.\",\n    )\n    parser.add_argument(\n        \"--n_best_size\",\n        type=int,\n        default=20,\n        help=\"The total number of n-best predictions to generate when looking for an answer.\",\n    )\n    parser.add_argument(\n        \"--null_score_diff_threshold\",\n        type=float,\n        default=0.0,\n        help=(\n            \"The threshold used to select the null answer: if the best answer has a score that is less than \"\n            \"the score of the null answer minus this threshold, the null answer is selected for this example. \"\n            \"Only useful when `version_2_with_negative=True`.\"\n        ),\n    )\n    parser.add_argument(\n        \"--version_2_with_negative\",\n        action=\"store_true\",\n        help=\"If true, some of the examples do not have an answer.\",\n    )\n    parser.add_argument(\n        \"--max_answer_length\",\n        type=int,\n        default=30,\n        help=(\n            \"The maximum length of an answer that can be generated. This is needed because the start \"\n            \"and end predictions are not conditioned on one another.\"\n        ),\n    )\n    parser.add_argument(\n        \"--max_train_samples\",\n        type=int,\n        default=None,\n        help=(\n            \"For debugging purposes or quicker training, truncate the number of training examples to this \"\n            \"value if set.\"\n        ),\n    )\n    parser.add_argument(\n        \"--max_eval_samples\",\n        type=int,\n        default=None,\n        help=(\n            \"For debugging purposes or quicker training, truncate the number of evaluation examples to this \"\n            \"value if set.\"\n        ),\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--max_predict_samples\",\n        type=int,\n        default=None,\n        help=\"For debugging purposes or quicker training, truncate the number of prediction examples to this\",\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to load in all available experiment trackers from the environment and use them for logging.\",\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if (\n        args.dataset_name is None\n        and args.train_file is None\n        and args.validation_file is None\n        and args.test_file is None\n    ):\n        raise ValueError(\"Need either a dataset name or a training/validation/test file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n        if args.test_file is not None:\n            extension = args.test_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`test_file` should be a csv or a json file.\"\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\n**Function Summary:**\n\nThis function, `parse_args()`, creates an ArgumentParser object and populates it with 35 optional arguments for fine-tuning a transformer model on a Question Answering task.\n\n**Purpose and Functionality:**\nThe primary purpose of this function is to parse command-line arguments for the fine-tuning process. It allows users to specify various parameters such as dataset name, model configuration, training data file, preprocessing configurations, and hyperparameters.\n\n**Key Implementation Details:**\n\n* The function supports both `dataset_name` and `train_file` modes of input for the first set of required arguments.\n* Various hyperparameters such as learning rate, batch size, number of epochs, and evaluation metrics are optional.\n* Additional parameters like `doc_stride`, `n_best_size`, and `null_score_diff_threshold` control specific aspects of the question answering system.\n\n**Overall:**\n\nThe function is designed to provide a flexible way to configure the fine-tuning process for transformer models on Question Answering datasets. It includes many options for customization, allowing users to tailor their experiments to suit their needs."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py:300-1062

metadata:
{"docstring": "", "end_line": 1062, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py", "name": "main", "start_line": 300, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_qa_beam_search_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        if args.test_file is not None:\n            data_files[\"test\"] = args.test_file\n            extension = args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files, field=\"data\")\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n\n    config = XLNetConfig.from_pretrained(args.model_name_or_path)\n    tokenizer = XLNetTokenizerFast.from_pretrained(args.model_name_or_path)\n    model = XLNetForQuestionAnswering.from_pretrained(\n        args.model_name_or_path, from_tf=bool(\".ckpt\" in args.model_name_or_path), config=config\n    )\n\n    # Preprocessing the datasets.\n    # Preprocessing is slightly different for training and evaluation.\n    column_names = raw_datasets[\"train\"].column_names\n\n    question_column_name = \"question\" if \"question\" in column_names else column_names[0]\n    context_column_name = \"context\" if \"context\" in column_names else column_names[1]\n    answer_column_name = \"answers\" if \"answers\" in column_names else column_names[2]\n\n    # Padding side determines if we do (question|context) or (context|question).\n    pad_on_right = tokenizer.padding_side == \"right\"\n\n    if args.max_seq_length > tokenizer.model_max_length:\n        logger.warning(\n            f\"The max_seq_length passed ({args.max_seq_length}) is larger than the maximum length for the \"\n            f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n        )\n\n    max_seq_length = min(args.max_seq_length, tokenizer.model_max_length)\n\n    # Training preprocessing\n    def prepare_train_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            return_special_tokens_mask=True,\n            return_token_type_ids=True,\n            padding=\"max_length\",\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n        # The offset mappings will give us a map from token to character position in the original context. This will\n        # help us compute the start_positions and end_positions.\n        offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n        # The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).\n        special_tokens = tokenized_examples.pop(\"special_tokens_mask\")\n\n        # Let's label those examples!\n        tokenized_examples[\"start_positions\"] = []\n        tokenized_examples[\"end_positions\"] = []\n        tokenized_examples[\"is_impossible\"] = []\n        tokenized_examples[\"cls_index\"] = []\n        tokenized_examples[\"p_mask\"] = []\n\n        for i, offsets in enumerate(offset_mapping):\n            # We will label impossible answers with the index of the CLS token.\n            input_ids = tokenized_examples[\"input_ids\"][i]\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n            tokenized_examples[\"cls_index\"].append(cls_index)\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples[\"token_type_ids\"][i]\n            for k, s in enumerate(special_tokens[i]):\n                if s:\n                    sequence_ids[k] = 3\n            context_idx = 1 if pad_on_right else 0\n\n            # Build the p_mask: non special tokens and context gets 0.0, the others get 1.0.\n            # The cls token gets 1.0 too (for predictions of empty answers).\n            tokenized_examples[\"p_mask\"].append(\n                [\n                    0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0\n                    for k, s in enumerate(sequence_ids)\n                ]\n            )\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            answers = examples[answer_column_name][sample_index]\n            # If no answers are given, set the cls_index as answer.\n            if len(answers[\"answer_start\"]) == 0:\n                tokenized_examples[\"start_positions\"].append(cls_index)\n                tokenized_examples[\"end_positions\"].append(cls_index)\n                tokenized_examples[\"is_impossible\"].append(1.0)\n            else:\n                # Start/end character index of the answer in the text.\n                start_char = answers[\"answer_start\"][0]\n                end_char = start_char + len(answers[\"text\"][0])\n\n                # Start token index of the current span in the text.\n                token_start_index = 0\n                while sequence_ids[token_start_index] != context_idx:\n                    token_start_index += 1\n\n                # End token index of the current span in the text.\n                token_end_index = len(input_ids) - 1\n                while sequence_ids[token_end_index] != context_idx:\n                    token_end_index -= 1\n                # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n                if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n                    tokenized_examples[\"start_positions\"].append(cls_index)\n                    tokenized_examples[\"end_positions\"].append(cls_index)\n                    tokenized_examples[\"is_impossible\"].append(1.0)\n                else:\n                    # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n                    # Note: we could go after the last offset if the answer is the last word (edge case).\n                    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n                        token_start_index += 1\n                    tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n                    while offsets[token_end_index][1] >= end_char:\n                        token_end_index -= 1\n                    tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n                    tokenized_examples[\"is_impossible\"].append(0.0)\n\n        return tokenized_examples\n\n    if \"train\" not in raw_datasets:\n        raise ValueError(\"--do_train requires a train dataset\")\n    train_dataset = raw_datasets[\"train\"]\n    if args.max_train_samples is not None:\n        # We will select sample from whole data if argument is specified\n        train_dataset = train_dataset.select(range(args.max_train_samples))\n    # Create train feature from dataset\n    with accelerator.main_process_first():\n        train_dataset = train_dataset.map(\n            prepare_train_features,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on train dataset\",\n        )\n    if args.max_train_samples is not None:\n        # Number of samples might increase during Feature Creation, We select only specified max samples\n        train_dataset = train_dataset.select(range(args.max_train_samples))\n\n    # Validation preprocessing\n    def prepare_validation_features(examples):\n        # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n        # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n        # left whitespace\n        examples[question_column_name] = [q.lstrip() for q in examples[question_column_name]]\n\n        # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n        # in one example possible giving several features when a context is long, each of those features having a\n        # context that overlaps a bit the context of the previous feature.\n        tokenized_examples = tokenizer(\n            examples[question_column_name if pad_on_right else context_column_name],\n            examples[context_column_name if pad_on_right else question_column_name],\n            truncation=\"only_second\" if pad_on_right else \"only_first\",\n            max_length=max_seq_length,\n            stride=args.doc_stride,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n            return_special_tokens_mask=True,\n            return_token_type_ids=True,\n            padding=\"max_length\",\n        )\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n\n        # The special tokens will help us build the p_mask (which indicates the tokens that can't be in answers).\n        special_tokens = tokenized_examples.pop(\"special_tokens_mask\")\n\n        # For evaluation, we will need to convert our predictions to substrings of the context, so we keep the\n        # corresponding example_id and we will store the offset mappings.\n        tokenized_examples[\"example_id\"] = []\n\n        # We still provide the index of the CLS token and the p_mask to the model, but not the is_impossible label.\n        tokenized_examples[\"cls_index\"] = []\n        tokenized_examples[\"p_mask\"] = []\n\n        for i, input_ids in enumerate(tokenized_examples[\"input_ids\"]):\n            # Find the CLS token in the input ids.\n            if tokenizer.cls_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.cls_token_id)\n            elif tokenizer.bos_token_id in input_ids:\n                cls_index = input_ids.index(tokenizer.bos_token_id)\n            else:\n                cls_index = 0\n            tokenized_examples[\"cls_index\"].append(cls_index)\n\n            # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n            sequence_ids = tokenized_examples[\"token_type_ids\"][i]\n            for k, s in enumerate(special_tokens[i]):\n                if s:\n                    sequence_ids[k] = 3\n            context_idx = 1 if pad_on_right else 0\n\n            # Build the p_mask: non special tokens and context gets 0.0, the others 1.0.\n            tokenized_examples[\"p_mask\"].append(\n                [\n                    0.0 if (not special_tokens[i][k] and s == context_idx) or k == cls_index else 1.0\n                    for k, s in enumerate(sequence_ids)\n                ]\n            )\n\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            tokenized_examples[\"example_id\"].append(examples[\"id\"][sample_index])\n\n            # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n            # position is part of the context or not.\n            tokenized_examples[\"offset_mapping\"][i] = [\n                (o if sequence_ids[k] == context_idx else None)\n                for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n            ]\n\n        return tokenized_examples\n\n    if \"validation\" not in raw_datasets:\n        raise ValueError(\"--do_eval requires a validation dataset\")\n    eval_examples = raw_datasets[\"validation\"]\n    if args.max_eval_samples is not None:\n        # We will select sample from whole data\n        eval_examples = eval_examples.select(range(args.max_eval_samples))\n    # Validation Feature Creation\n    with accelerator.main_process_first():\n        eval_dataset = eval_examples.map(\n            prepare_validation_features,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on validation dataset\",\n        )\n\n    if args.max_eval_samples is not None:\n        # During Feature creation dataset samples might increase, we will select required samples again\n        eval_dataset = eval_dataset.select(range(args.max_eval_samples))\n\n    if args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_examples = raw_datasets[\"test\"]\n        if args.max_predict_samples is not None:\n            # We will select sample from whole data\n            predict_examples = predict_examples.select(range(args.max_predict_samples))\n        # Predict Feature Creation\n        with accelerator.main_process_first():\n            predict_dataset = predict_examples.map(\n                prepare_validation_features,\n                batched=True,\n                num_proc=args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n            if args.max_predict_samples is not None:\n                # During Feature creation dataset samples might increase, we will select required samples again\n                predict_dataset = predict_dataset.select(range(args.max_predict_samples))\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    if args.pad_to_max_length:\n        # If padding was already done ot max length, we use the default data collator that will just convert everything\n        # to tensors.\n        data_collator = default_data_collator\n    else:\n        # Otherwise, `DataCollatorWithPadding` will apply dynamic padding for us (by padding to the maximum length of\n        # the samples passed). When using mixed precision, we add `pad_to_multiple_of=8` to pad all tensors to multiple\n        # of 8s, which will enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\n        # For fp8, we pad to multiple of 16.\n        if accelerator.mixed_precision == \"fp8\":\n            pad_to_multiple_of = 16\n        elif accelerator.mixed_precision != \"no\":\n            pad_to_multiple_of = 8\n        else:\n            pad_to_multiple_of = None\n        data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=pad_to_multiple_of)\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n\n    eval_dataset_for_model = eval_dataset.remove_columns([\"example_id\", \"offset_mapping\"])\n    eval_dataloader = DataLoader(\n        eval_dataset_for_model, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size\n    )\n\n    if args.do_predict:\n        predict_dataset_for_model = predict_dataset.remove_columns([\"example_id\", \"offset_mapping\"])\n        predict_dataloader = DataLoader(\n            predict_dataset_for_model, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size\n        )\n\n    # Post-processing:\n    def post_processing_function(examples, features, predictions, stage=\"eval\"):\n        # Post-processing: we match the start logits and end logits to answers in the original context.\n        predictions, scores_diff_json = postprocess_qa_predictions_with_beam_search(\n            examples=examples,\n            features=features,\n            predictions=predictions,\n            version_2_with_negative=args.version_2_with_negative,\n            n_best_size=args.n_best_size,\n            max_answer_length=args.max_answer_length,\n            start_n_top=model.config.start_n_top,\n            end_n_top=model.config.end_n_top,\n            output_dir=args.output_dir,\n            prefix=stage,\n        )\n        # Format the result to the format the metric expects.\n        if args.version_2_with_negative:\n            formatted_predictions = [\n                {\"id\": k, \"prediction_text\": v, \"no_answer_probability\": scores_diff_json[k]}\n                for k, v in predictions.items()\n            ]\n        else:\n            formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in predictions.items()]\n\n        references = [{\"id\": ex[\"id\"], \"answers\": ex[answer_column_name]} for ex in examples]\n        return EvalPrediction(predictions=formatted_predictions, label_ids=references)\n\n    metric = evaluate.load(\"squad_v2\" if args.version_2_with_negative else \"squad\")\n\n    def create_and_fill_np_array(start_or_end_logits, dataset, max_len):\n        \"\"\"\n        Create and fill numpy array of size len_of_validation_data * max_length_of_output_tensor\n\n        Args:\n            start_or_end_logits(:obj:`tensor`):\n                This is the output predictions of the model. We can only enter either start or end logits.\n            eval_dataset: Evaluation dataset\n            max_len(:obj:`int`):\n                The maximum length of the output tensor. ( See the model.eval() part for more details )\n        \"\"\"\n\n        step = 0\n        # create a numpy array and fill it with -100.\n        logits_concat = np.full((len(dataset), max_len), -100, dtype=np.float32)\n        # Now since we have create an array now we will populate it with the outputs gathered using accelerator.gather_for_metrics\n        for i, output_logit in enumerate(start_or_end_logits):  # populate columns\n            # We have to fill it such that we have to take the whole tensor and replace it on the newly created array\n            # And after every iteration we have to change the step\n\n            batch_size = output_logit.shape[0]\n            cols = output_logit.shape[1]\n            if step + batch_size < len(dataset):\n                logits_concat[step : step + batch_size, :cols] = output_logit\n            else:\n                logits_concat[step:, :cols] = output_logit[: len(dataset) - step]\n\n            step += batch_size\n\n        return logits_concat\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"qa_beam_search_no_trainer\", experiment_config)\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n\n                accelerator.backward(loss)\n\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    accelerator.save_state(f\"step_{completed_steps}\")\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n    # initialize all lists to collect the batches\n    all_start_top_log_probs = []\n    all_start_top_index = []\n    all_end_top_log_probs = []\n    all_end_top_index = []\n    all_cls_logits = []\n\n    model.eval()\n\n    for step, batch in enumerate(eval_dataloader):\n        with torch.no_grad():\n            outputs = model(**batch)\n            start_top_log_probs = outputs.start_top_log_probs\n            start_top_index = outputs.start_top_index\n            end_top_log_probs = outputs.end_top_log_probs\n            end_top_index = outputs.end_top_index\n            cls_logits = outputs.cls_logits\n\n            if not args.pad_to_max_length:  # necessary to pad predictions and labels for being gathered\n                start_top_log_probs = accelerator.pad_across_processes(start_top_log_probs, dim=1, pad_index=-100)\n                start_top_index = accelerator.pad_across_processes(start_top_index, dim=1, pad_index=-100)\n                end_top_log_probs = accelerator.pad_across_processes(end_top_log_probs, dim=1, pad_index=-100)\n                end_top_index = accelerator.pad_across_processes(end_top_index, dim=1, pad_index=-100)\n                cls_logits = accelerator.pad_across_processes(cls_logits, dim=1, pad_index=-100)\n\n            all_start_top_log_probs.append(accelerator.gather_for_metrics(start_top_log_probs).cpu().numpy())\n            all_start_top_index.append(accelerator.gather_for_metrics(start_top_index).cpu().numpy())\n            all_end_top_log_probs.append(accelerator.gather_for_metrics(end_top_log_probs).cpu().numpy())\n            all_end_top_index.append(accelerator.gather_for_metrics(end_top_index).cpu().numpy())\n            all_cls_logits.append(accelerator.gather_for_metrics(cls_logits).cpu().numpy())\n\n    max_len = max([x.shape[1] for x in all_end_top_log_probs])  # Get the max_length of the tensor\n\n    # concatenate all numpy arrays collected above\n    start_top_log_probs_concat = create_and_fill_np_array(all_start_top_log_probs, eval_dataset, max_len)\n    start_top_index_concat = create_and_fill_np_array(all_start_top_index, eval_dataset, max_len)\n    end_top_log_probs_concat = create_and_fill_np_array(all_end_top_log_probs, eval_dataset, max_len)\n    end_top_index_concat = create_and_fill_np_array(all_end_top_index, eval_dataset, max_len)\n    cls_logits_concat = np.concatenate(all_cls_logits, axis=0)\n\n    # delete the list of numpy arrays\n    del start_top_log_probs\n    del start_top_index\n    del end_top_log_probs\n    del end_top_index\n    del cls_logits\n\n    outputs_numpy = (\n        start_top_log_probs_concat,\n        start_top_index_concat,\n        end_top_log_probs_concat,\n        end_top_index_concat,\n        cls_logits_concat,\n    )\n    prediction = post_processing_function(eval_examples, eval_dataset, outputs_numpy)\n    eval_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)\n    logger.info(f\"Evaluation metrics: {eval_metric}\")\n\n    if args.do_predict:\n        # initialize all lists to collect the batches\n\n        all_start_top_log_probs = []\n        all_start_top_index = []\n        all_end_top_log_probs = []\n        all_end_top_index = []\n        all_cls_logits = []\n\n        model.eval()\n\n        for step, batch in enumerate(predict_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n                start_top_log_probs = outputs.start_top_log_probs\n                start_top_index = outputs.start_top_index\n                end_top_log_probs = outputs.end_top_log_probs\n                end_top_index = outputs.end_top_index\n                cls_logits = outputs.cls_logits\n\n                if not args.pad_to_max_length:  # necessary to pad predictions and labels for being gathered\n                    start_top_log_probs = accelerator.pad_across_processes(start_top_log_probs, dim=1, pad_index=-100)\n                    start_top_index = accelerator.pad_across_processes(start_top_index, dim=1, pad_index=-100)\n                    end_top_log_probs = accelerator.pad_across_processes(end_top_log_probs, dim=1, pad_index=-100)\n                    end_top_index = accelerator.pad_across_processes(end_top_index, dim=1, pad_index=-100)\n                    cls_logits = accelerator.pad_across_processes(cls_logits, dim=1, pad_index=-100)\n\n                all_start_top_log_probs.append(accelerator.gather_for_metrics(start_top_log_probs).cpu().numpy())\n                all_start_top_index.append(accelerator.gather_for_metrics(start_top_index).cpu().numpy())\n                all_end_top_log_probs.append(accelerator.gather_for_metrics(end_top_log_probs).cpu().numpy())\n                all_end_top_index.append(accelerator.gather_for_metrics(end_top_index).cpu().numpy())\n                all_cls_logits.append(accelerator.gather_for_metrics(cls_logits).cpu().numpy())\n\n        max_len = max([x.shape[1] for x in all_end_top_log_probs])  # Get the max_length of the tensor\n\n        # concatenate all numpy arrays collected above\n        start_top_log_probs_concat = create_and_fill_np_array(all_start_top_log_probs, predict_dataset, max_len)\n        start_top_index_concat = create_and_fill_np_array(all_start_top_index, predict_dataset, max_len)\n        end_top_log_probs_concat = create_and_fill_np_array(all_end_top_log_probs, predict_dataset, max_len)\n        end_top_index_concat = create_and_fill_np_array(all_end_top_index, predict_dataset, max_len)\n        cls_logits_concat = np.concatenate(all_cls_logits, axis=0)\n\n        # delete the list of numpy arrays\n        del start_top_log_probs\n        del start_top_index\n        del end_top_log_probs\n        del end_top_index\n        del cls_logits\n\n        outputs_numpy = (\n            start_top_log_probs_concat,\n            start_top_index_concat,\n            end_top_log_probs_concat,\n            end_top_index_concat,\n            cls_logits_concat,\n        )\n\n        prediction = post_processing_function(predict_examples, predict_dataset, outputs_numpy)\n        predict_metric = metric.compute(predictions=prediction.predictions, references=prediction.label_ids)\n        logger.info(f\"Predict metrics: {predict_metric}\")\n\n    if args.with_tracking:\n        log = {\n            \"squad_v2\" if args.version_2_with_negative else \"squad\": eval_metric,\n            \"train_loss\": total_loss,\n            \"epoch\": epoch,\n            \"step\": completed_steps,\n        }\n        if args.do_predict:\n            log[\"squad_v2_predict\" if args.version_2_with_negative else \"squad_predict\"] = predict_metric\n\n        accelerator.log(log)\n\n    if args.checkpointing_steps == \"epoch\":\n        accelerator.save_state(f\"epoch_{epoch}\")\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n            logger.info(json.dumps(eval_metric, indent=4))\n\n\nSUMMARY:\nHere is a concise summary of the code:\n\nThis is an implementation of the DistilBERT model on the Stanford Question Answering Dataset (SQuAD). The code trains the model using Adam optimizer and cross-entropy loss, and evaluates its performance on both training and prediction datasets. It also tracks various metrics, including train loss, epoch, step, and prediction accuracy, and logs them to a log file.\n\nKey features:\n\n* Training and evaluation of DistilBERT model on SQuAD\n* Adam optimizer with cross_entropy loss\n* Tracking of training and prediction performance metrics (train_loss, epoch, step, predict_accuracy)\n* Logging of metrics to a log file\n* Options for pushing the model to a hub repository after training\n\nThe code is written in Python and uses the Hugging Face Transformers library for the DistilBERT model. It appears to be designed for use with popular logging frameworks such as TensorBoard or Summarize, although this is not explicitly mentioned in the code."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/trainer_seq2seq_qa.py:33-163

metadata:
{"docstring": "", "end_line": 163, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/trainer_seq2seq_qa.py", "name": "QuestionAnsweringSeq2SeqTrainer", "start_line": 33, "type": "class", "chroma:document": "CODE:\n\nclass QuestionAnsweringSeq2SeqTrainer(Seq2SeqTrainer):\n    def __init__(self, *args, eval_examples=None, post_process_function=None, **kwargs):\n        super().__init__(*args, **kwargs)\n        self.eval_examples = eval_examples\n        self.post_process_function = post_process_function\n\n    # def evaluate(self, eval_dataset=None, eval_examples=None, ignore_keys=None, metric_key_prefix: str = \"eval\"):\n    def evaluate(\n        self,\n        eval_dataset: Optional[Dataset] = None,\n        eval_examples=None,\n        ignore_keys: Optional[List[str]] = None,\n        metric_key_prefix: str = \"eval\",\n        **gen_kwargs,\n    ) -> Dict[str, float]:\n        gen_kwargs = gen_kwargs.copy()\n\n        # Use legacy argument setting if a) the option is not explicitly passed; and b) the argument is set in the\n        # training args\n        if gen_kwargs.get(\"max_length\") is None and self.args.generation_max_length is not None:\n            gen_kwargs[\"max_length\"] = self.args.generation_max_length\n        if gen_kwargs.get(\"num_beams\") is None and self.args.generation_num_beams is not None:\n            gen_kwargs[\"num_beams\"] = self.args.generation_num_beams\n        self._gen_kwargs = gen_kwargs\n\n        eval_dataset = self.eval_dataset if eval_dataset is None else eval_dataset\n        eval_dataloader = self.get_eval_dataloader(eval_dataset)\n        eval_examples = self.eval_examples if eval_examples is None else eval_examples\n\n        # Temporarily disable metric computation, we will do it in the loop here.\n        compute_metrics = self.compute_metrics\n        self.compute_metrics = None\n        start_time = time.time()\n        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop\n        try:\n            output = eval_loop(\n                eval_dataloader,\n                description=\"Evaluation\",\n                # No point gathering the predictions if there are no metrics, otherwise we defer to\n                # self.args.prediction_loss_only\n                prediction_loss_only=True if compute_metrics is None else None,\n                ignore_keys=ignore_keys,\n                metric_key_prefix=metric_key_prefix,\n            )\n        finally:\n            self.compute_metrics = compute_metrics\n        total_batch_size = self.args.eval_batch_size * self.args.world_size\n        if f\"{metric_key_prefix}_jit_compilation_time\" in output.metrics:\n            start_time += output.metrics[f\"{metric_key_prefix}_jit_compilation_time\"]\n        output.metrics.update(\n            speed_metrics(\n                metric_key_prefix,\n                start_time,\n                num_samples=output.num_samples,\n                num_steps=math.ceil(output.num_samples / total_batch_size),\n            )\n        )\n\n        if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:\n            # Only the main node write the results by default\n            eval_preds = self.post_process_function(eval_examples, eval_dataset, output)\n            metrics = self.compute_metrics(eval_preds)\n\n            # Prefix all keys with metric_key_prefix + '_'\n            for key in list(metrics.keys()):\n                if not key.startswith(f\"{metric_key_prefix}_\"):\n                    metrics[f\"{metric_key_prefix}_{key}\"] = metrics.pop(key)\n\n            metrics.update(output.metrics)\n        else:\n            metrics = output.metrics\n\n        if self.args.should_log:\n            # Only the main node log the results by default\n            self.log(metrics)\n\n        if self.args.tpu_metrics_debug or self.args.debug:\n            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)\n            xm.master_print(met.metrics_report())\n\n        self.control = self.callback_handler.on_evaluate(self.args, self.state, self.control, metrics)\n        return metrics\n\n    def predict(\n        self, predict_dataset, predict_examples, ignore_keys=None, metric_key_prefix: str = \"test\", **gen_kwargs\n    ):\n        self._gen_kwargs = gen_kwargs.copy()\n\n        predict_dataloader = self.get_test_dataloader(predict_dataset)\n\n        # Temporarily disable metric computation, we will do it in the loop here.\n        compute_metrics = self.compute_metrics\n        self.compute_metrics = None\n        start_time = time.time()\n        eval_loop = self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop\n        try:\n            output = eval_loop(\n                predict_dataloader,\n                description=\"Prediction\",\n                # No point gathering the predictions if there are no metrics, otherwise we defer to\n                # self.args.prediction_loss_only\n                prediction_loss_only=True if compute_metrics is None else None,\n                ignore_keys=ignore_keys,\n                metric_key_prefix=metric_key_prefix,\n            )\n        finally:\n            self.compute_metrics = compute_metrics\n\n        total_batch_size = self.args.eval_batch_size * self.args.world_size\n        if f\"{metric_key_prefix}_jit_compilation_time\" in output.metrics:\n            start_time += output.metrics[f\"{metric_key_prefix}_jit_compilation_time\"]\n        output.metrics.update(\n            speed_metrics(\n                metric_key_prefix,\n                start_time,\n                num_samples=output.num_samples,\n                num_steps=math.ceil(output.num_samples / total_batch_size),\n            )\n        )\n        if self.post_process_function is None or self.compute_metrics is None:\n            return output\n\n        predictions = self.post_process_function(predict_examples, predict_dataset, output, \"predict\")\n        metrics = self.compute_metrics(predictions)\n\n        # Prefix all keys with metric_key_prefix + '_'\n        for key in list(metrics.keys()):\n            if not key.startswith(f\"{metric_key_prefix}_\"):\n                metrics[f\"{metric_key_prefix}_{key}\"] = metrics.pop(key)\n        metrics.update(output.metrics)\n\n\nSUMMARY:\n**Summary:**\n\nThis is an implementation of the `QuestionAnsweringSeq2SeqTrainer` class in Hugging Face Transformers. It inherits from `Seq2SeqTrainer` and provides functionality for training and evaluating question-answering models using sequence-to-sequence learning.\n\n**Main Functionality:**\n\n1. Evaluates a model on a dataset using the `evaluate` method.\n2. Generates predictions on a test dataset using the `predict` method.\n3. Preprocesses and post-processes input data for evaluation or prediction.\n4. Computation of metrics, such as speed, compilation time, and loss.\n\n**Key Implementation Details:**\n\n* `evaluate`: uses `prediction_loop` or `evaluation_loop`, disables metric computation temporarily, and updates output metrics with speed statistics.\n* `predict`: uses `prediction_loop` or `evaluation_loop`, generates predictions using `post_process_function`, and computes metrics on the prediction output.\n\n**Notable Dependencies:**\n\n* Hugging Face Transformers library\n* PyTorch library\n\nExternal dependencies:\n\n* None\n\nInterfaces:\n\n* `Seq2SeqTrainer`\n* `Dataset`\n* `PredictionOutput`\n\nThis summary provides a concise overview of the key functionality, implementation details, and dependencies of the provided class code chunk."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_seq2seq_qa.py:272-733

metadata:
{"docstring": "", "end_line": 733, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_seq2seq_qa.py", "name": "main", "start_line": 272, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_seq2seq_qa\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(\n            extension,\n            data_files=data_files,\n            field=\"data\",\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        use_fast=model_args.use_fast_tokenizer,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    if model.config.decoder_start_token_id is None:\n        raise ValueError(\"Make sure that `config.decoder_start_token_id` is correctly defined\")\n\n    # Preprocessing the datasets.\n    # We need to generate and tokenize inputs and targets.\n    if training_args.do_train:\n        column_names = raw_datasets[\"train\"].column_names\n    elif training_args.do_eval:\n        column_names = raw_datasets[\"validation\"].column_names\n    elif training_args.do_predict:\n        column_names = raw_datasets[\"test\"].column_names\n    else:\n        logger.info(\"There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.\")\n        return\n\n    # Get the column names for input/target.\n    dataset_columns = question_answering_column_name_mapping.get(data_args.dataset_name, None)\n    if data_args.question_column is None:\n        question_column = dataset_columns[0] if dataset_columns is not None else column_names[0]\n    else:\n        question_column = data_args.question_column\n        if question_column not in column_names:\n            raise ValueError(\n                f\"--question_column' value '{data_args.question_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n    if data_args.context_column is None:\n        context_column = dataset_columns[1] if dataset_columns is not None else column_names[1]\n    else:\n        context_column = data_args.context_column\n        if context_column not in column_names:\n            raise ValueError(\n                f\"--context_column' value '{data_args.context_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n    if data_args.answer_column is None:\n        answer_column = dataset_columns[2] if dataset_columns is not None else column_names[2]\n    else:\n        answer_column = data_args.answer_column\n        if answer_column not in column_names:\n            raise ValueError(\n                f\"--answer_column' value '{data_args.answer_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n\n    # Temporarily set max_answer_length for training.\n    max_answer_length = data_args.max_answer_length\n    padding = \"max_length\" if data_args.pad_to_max_length else False\n\n    if training_args.label_smoothing_factor > 0 and not hasattr(model, \"prepare_decoder_input_ids_from_labels\"):\n        logger.warning(\n            \"label_smoothing is enabled but the `prepare_decoder_input_ids_from_labels` method is not defined for \"\n            f\"`{model.__class__.__name__}`. This will lead to loss being calculated twice and will take up more memory\"\n        )\n\n    if data_args.max_seq_length > tokenizer.model_max_length:\n        logger.warning(\n            f\"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the \"\n            f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n        )\n    max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)\n\n    def preprocess_squad_batch(\n        examples,\n        question_column: str,\n        context_column: str,\n        answer_column: str,\n    ) -> Tuple[List[str], List[str]]:\n        questions = examples[question_column]\n        contexts = examples[context_column]\n        answers = examples[answer_column]\n\n        def generate_input(_question, _context):\n            return \" \".join([\"question:\", _question.lstrip(), \"context:\", _context.lstrip()])\n\n        inputs = [generate_input(question, context) for question, context in zip(questions, contexts)]\n        targets = [answer[\"text\"][0] if len(answer[\"text\"]) > 0 else \"\" for answer in answers]\n        return inputs, targets\n\n    def preprocess_function(examples):\n        inputs, targets = preprocess_squad_batch(examples, question_column, context_column, answer_column)\n\n        model_inputs = tokenizer(inputs, max_length=max_seq_length, padding=padding, truncation=True)\n        # Tokenize targets with text_target=...\n        labels = tokenizer(text_target=targets, max_length=max_answer_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and data_args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    # Validation preprocessing\n    def preprocess_validation_function(examples):\n        inputs, targets = preprocess_squad_batch(examples, question_column, context_column, answer_column)\n\n        model_inputs = tokenizer(\n            inputs,\n            max_length=max_seq_length,\n            padding=padding,\n            truncation=True,\n            return_overflowing_tokens=True,\n            return_offsets_mapping=True,\n        )\n        # Tokenize targets with the `text_target` keyword argument\n        labels = tokenizer(text_target=targets, max_length=max_answer_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and data_args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        # Since one example might give us several features if it has a long context, we need a map from a feature to\n        # its corresponding example. This key gives us just that.\n        sample_mapping = model_inputs.pop(\"overflow_to_sample_mapping\")\n\n        # For evaluation, we will need to convert our predictions to substrings of the context, so we keep the\n        # corresponding example_id and we will store the offset mappings.\n        model_inputs[\"example_id\"] = []\n        # Augment the overflowing tokens to the labels\n        labels_out = []\n\n        for i in range(len(model_inputs[\"input_ids\"])):\n            # One example can give several spans, this is the index of the example containing this span of text.\n            sample_index = sample_mapping[i]\n            model_inputs[\"example_id\"].append(examples[\"id\"][sample_index])\n            labels_out.append(labels[\"input_ids\"][sample_index])\n\n        model_inputs[\"labels\"] = labels_out\n        return model_inputs\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            # We will select sample from whole data if argument is specified\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        # Create train feature from dataset\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n        if data_args.max_train_samples is not None:\n            # Number of samples might increase during Feature Creation, We select only specified max samples\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n\n    if training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_examples = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            # We will select sample from whole data\n            max_eval_samples = min(len(eval_examples), data_args.max_eval_samples)\n            eval_examples = eval_examples.select(range(max_eval_samples))\n        # Validation Feature Creation\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_examples.map(\n                preprocess_validation_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n        if data_args.max_eval_samples is not None:\n            # During Feature creation dataset samples might increase, we will select required samples again\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n\n    if training_args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_examples = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            # We will select sample from whole data\n            predict_examples = predict_examples.select(range(data_args.max_predict_samples))\n        # Predict Feature Creation\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_examples.map(\n                preprocess_validation_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n        if data_args.max_predict_samples is not None:\n            # During Feature creation dataset samples might increase, we will select required samples again\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n\n    # Data collator\n    label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n    data_collator = DataCollatorForSeq2Seq(\n        tokenizer,\n        model=model,\n        label_pad_token_id=label_pad_token_id,\n        pad_to_multiple_of=8 if training_args.fp16 else None,\n    )\n\n    metric = evaluate.load(\n        \"squad_v2\" if data_args.version_2_with_negative else \"squad\", cache_dir=model_args.cache_dir\n    )\n\n    def compute_metrics(p: EvalPrediction):\n        return metric.compute(predictions=p.predictions, references=p.label_ids)\n\n    # Post-processing:\n    def post_processing_function(\n        examples: datasets.Dataset, features: datasets.Dataset, outputs: EvalLoopOutput, stage=\"eval\"\n    ):\n        # Decode the predicted tokens.\n        preds = outputs.predictions\n        if isinstance(preds, tuple):\n            preds = preds[0]\n        # Replace -100s used for padding as we can't decode them\n        preds = np.where(preds != -100, preds, tokenizer.pad_token_id)\n        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n\n        # Build a map example to its corresponding features.\n        example_id_to_index = {k: i for i, k in enumerate(examples[\"id\"])}\n        feature_per_example = {example_id_to_index[feature[\"example_id\"]]: i for i, feature in enumerate(features)}\n        predictions = {}\n        # Let's loop over all the examples!\n        for example_index, example in enumerate(examples):\n            # This is the index of the feature associated to the current example.\n            feature_index = feature_per_example[example_index]\n            predictions[example[\"id\"]] = decoded_preds[feature_index]\n\n        # Format the result to the format the metric expects.\n        if data_args.version_2_with_negative:\n            formatted_predictions = [\n                {\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in predictions.items()\n            ]\n        else:\n            formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in predictions.items()]\n\n        references = [{\"id\": ex[\"id\"], \"answers\": ex[answer_column]} for ex in examples]\n        return EvalPrediction(predictions=formatted_predictions, label_ids=references)\n\n    # Initialize our Trainer\n    trainer = QuestionAnsweringSeq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        eval_examples=eval_examples if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics if training_args.predict_with_generate else None,\n        post_process_function=post_processing_function,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        metrics = train_result.metrics\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    results = {}\n    max_length = (\n        training_args.generation_max_length\n        if training_args.generation_max_length is not None\n        else data_args.val_max_answer_length\n    )\n    num_beams = data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n        metrics = trainer.evaluate(max_length=max_length, num_beams=num_beams, metric_key_prefix=\"eval\")\n\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Prediction\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n        results = trainer.predict(predict_dataset, predict_examples)\n        metrics = results.metrics\n\n        max_predict_samples = (\n            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n        )\n        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n    if training_args.push_to_hub:\n        kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"question-answering\"}\n        if data_args.dataset_name is not None:\n            kwargs[\"dataset_tags\"] = data_args.dataset_name\n            if data_args.dataset_config_name is not None:\n                kwargs[\"dataset_args\"] = data_args.dataset_config_name\n                kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n            else:\n                kwargs[\"dataset\"] = data_args.dataset_name\n\n\n\nSUMMARY:\nThis code is for training and evaluating a Question Answering (QA) model using the Hugging Face Transformers library. It includes dataset processing, model training, evaluation, prediction, and model pushing to a hub.\n\nHere's a high-level overview:\n\n1. **Dataset Preprocessing**: The code preprocesses the QA data by applying tokenization, label encoding, and padding.\n2. **Model Training**: It trains the model using the `QuestionAnsweringSeq2SeqTrainer` class, which combines sequence-to-sequence training with loss-based optimization.\n3. **Evaluating Performance**: During evaluation, the code assesses the model's performance on a validation set using various metrics (e.g., accuracy, precision, recall).\n4. **Predicting Answers**: The model uses its trained weights to predict answers for input questions.\n5. **Saving and Pushing Model**: After training or prediction, the model is saved and, optionally, pushed to a Model Hub.\n\nThe code also includes several hyperparameters that can be adjusted (e.g., `num_beams`, `max_length`, `dataset_name`).\n\n**Key Features:**\n\n* Supports both traditional and sequence-to-sequence training\n* Uses Hugging Face's Transformers library for tokenization and optimization\n* Includes post-processing for answer decoding and evaluation metric calculation"}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_seq2seq_qa.py:736-738

metadata:
{"docstring": "", "end_line": 738, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/question-answering/run_seq2seq_qa.py", "name": "_mp_fn", "start_line": 736, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Summary:**\n\n### Function Analysis\n\n* Name: `_mp_fn`\n* Purpose: Executes the `main()` function, likely as part of parallel processing or multiprocessing using TPUs via `xla_spawn`.\n\n* Implementation Details:\n\t+ Calls the outer-scoped `main()` function directly.\n\n### Notable Dependencies and Requirements:\n\n* `xla_spawn` for TPU execution\n* Assumed availability of an outer-scoped `main()` function."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization_no_trainer.py:92-320

metadata:
{"docstring": "", "end_line": 320, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization_no_trainer.py", "name": "parse_args", "start_line": 92, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a summarization task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--ignore_pad_token_for_loss\",\n        type=bool,\n        default=True,\n        help=\"Whether to ignore the tokens corresponding to padded labels in the loss computation or not.\",\n    )\n    parser.add_argument(\n        \"--max_source_length\",\n        type=int,\n        default=1024,\n        help=(\n            \"The maximum total input sequence length after \"\n            \"tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.\"\n        ),\n    )\n    parser.add_argument(\n        \"--source_prefix\",\n        type=str,\n        default=None,\n        help=\"A prefix to add before every source text (useful for T5 models).\",\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\",\n        type=int,\n        default=None,\n        help=\"The number of processes to use for the preprocessing.\",\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--max_target_length\",\n        type=int,\n        default=128,\n        help=(\n            \"The maximum total sequence length for target text after \"\n            \"tokenization. Sequences longer than this will be truncated, sequences shorter will be padded. \"\n            \"during ``evaluate`` and ``predict``.\"\n        ),\n    )\n    parser.add_argument(\n        \"--val_max_target_length\",\n        type=int,\n        default=None,\n        help=(\n            \"The maximum total sequence length for validation \"\n            \"target text after tokenization.Sequences longer than this will be truncated, sequences shorter will be \"\n            \"padded. Will default to `max_target_length`.This argument is also used to override the ``max_length`` \"\n            \"param of ``model.generate``, which is used during ``evaluate`` and ``predict``.\"\n        ),\n    )\n    parser.add_argument(\n        \"--num_beams\",\n        type=int,\n        default=None,\n        help=(\n            \"Number of beams to use for evaluation. This argument will be \"\n            \"passed to ``model.generate``, which is used during ``evaluate`` and ``predict``.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--text_column\",\n        type=str,\n        default=None,\n        help=\"The name of the column in the datasets containing the full texts (for summarization).\",\n    )\n    parser.add_argument(\n        \"--summary_column\",\n        type=str,\n        default=None,\n        help=\"The name of the column in the datasets containing the summaries (for summarization).\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.dataset_name is None and args.train_file is None and args.validation_file is None:\n        raise ValueError(\"Need either a dataset name or a training/validation file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\n**Function Summary**\n\nThis function parses command-line arguments for the Hugging Face Transformers library. The function takes in various hyperparameters and optimization settings for training a model, including:\n\n* Model architecture parameters\n* Training data configuration\n* Hyperparameter tunables (e.g., learning rate, weight decay)\n* Optimization settings (e.g., scheduler type, accumulation steps)\n\nThe function returns a dictionary of parsed arguments.\n\n**Key Parameters**\n\n* `--train_data`: Training dataset file(s) or dataset name\n* `--validation_data`: Validation dataset file(s) or dataset name\n* `--output_dir`: Directory for saving the trained model\n* `--learning_rate`: Initial learning rate\n* `--num_train_epochs`: Total number of training epochs\n* `--scheduler_type`: Optimization scheduler type (e.g., linear, cosine)\n* `--gradient_accumulation_steps`: Accumulation steps for gradient updates\n\n**Return Value**\n\nA dictionary containing the parsed command-line arguments."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization_no_trainer.py:323-796

metadata:
{"docstring": "", "end_line": 796, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization_no_trainer.py", "name": "main", "start_line": 323, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_summarization_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n    if args.source_prefix is None and args.model_name_or_path in [\n        \"google-t5/t5-small\",\n        \"google-t5/t5-base\",\n        \"google-t5/t5-large\",\n        \"google-t5/t5-3b\",\n        \"google-t5/t5-11b\",\n    ]:\n        logger.warning(\n            \"You're running a t5 model but didn't provide a source prefix, which is the expected, e.g. with \"\n            \"`--source_prefix 'summarize: ' `\"\n        )\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files)\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(args.config_name, trust_remote_code=args.trust_remote_code)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForSeq2SeqLM.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForSeq2SeqLM.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n    if model.config.decoder_start_token_id is None:\n        raise ValueError(\"Make sure that `config.decoder_start_token_id` is correctly defined\")\n\n    prefix = args.source_prefix if args.source_prefix is not None else \"\"\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    column_names = raw_datasets[\"train\"].column_names\n\n    # Get the column names for input/target.\n    dataset_columns = summarization_name_mapping.get(args.dataset_name, None)\n    if args.text_column is None:\n        text_column = dataset_columns[0] if dataset_columns is not None else column_names[0]\n    else:\n        text_column = args.text_column\n        if text_column not in column_names:\n            raise ValueError(\n                f\"--text_column' value '{args.text_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n    if args.summary_column is None:\n        summary_column = dataset_columns[1] if dataset_columns is not None else column_names[1]\n    else:\n        summary_column = args.summary_column\n        if summary_column not in column_names:\n            raise ValueError(\n                f\"--summary_column' value '{args.summary_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n\n    if args.val_max_target_length is None:\n        args.val_max_target_length = args.max_target_length\n\n    # Temporarily set max_target_length for training.\n    max_target_length = args.max_target_length\n    padding = \"max_length\" if args.pad_to_max_length else False\n\n    def preprocess_function(examples):\n        inputs = examples[text_column]\n        targets = examples[summary_column]\n        inputs = [prefix + inp for inp in inputs]\n        model_inputs = tokenizer(inputs, max_length=args.max_source_length, padding=padding, truncation=True)\n\n        # Tokenize targets with the `text_target` keyword argument\n        labels = tokenizer(text_target=targets, max_length=max_target_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    with accelerator.main_process_first():\n        train_dataset = raw_datasets[\"train\"].map(\n            preprocess_function,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on dataset\",\n        )\n\n        # Temporarily set max_target_length for validation.\n        max_target_length = args.val_max_target_length\n        eval_dataset = raw_datasets[\"validation\"].map(\n            preprocess_function,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on dataset\",\n        )\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 1):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    label_pad_token_id = -100 if args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n    if accelerator.mixed_precision == \"fp8\":\n        pad_to_multiple_of = 16\n    elif accelerator.mixed_precision != \"no\":\n        pad_to_multiple_of = 8\n    else:\n        pad_to_multiple_of = None\n    data_collator = DataCollatorForSeq2Seq(\n        tokenizer,\n        model=model,\n        label_pad_token_id=label_pad_token_id,\n        pad_to_multiple_of=pad_to_multiple_of,\n    )\n\n    def postprocess_text(preds, labels):\n        preds = [pred.strip() for pred in preds]\n        labels = [label.strip() for label in labels]\n\n        # rougeLSum expects newline after each sentence\n        preds = [\"\\n\".join(nltk.sent_tokenize(pred)) for pred in preds]\n        labels = [\"\\n\".join(nltk.sent_tokenize(label)) for label in labels]\n\n        return preds, labels\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\", \"layer_norm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"summarization_no_trainer\", experiment_config)\n\n    # Metric\n    metric = evaluate.load(\"rouge\")\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n\n        gen_kwargs = {\n            \"max_length\": args.val_max_target_length,\n            \"num_beams\": args.num_beams,\n        }\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                generated_tokens = accelerator.unwrap_model(model).generate(\n                    batch[\"input_ids\"],\n                    attention_mask=batch[\"attention_mask\"],\n                    **gen_kwargs,\n                )\n\n                generated_tokens = accelerator.pad_across_processes(\n                    generated_tokens, dim=1, pad_index=tokenizer.pad_token_id\n                )\n                labels = batch[\"labels\"]\n                if not args.pad_to_max_length:\n                    # If we did not pad to max length, we need to pad the labels too\n                    labels = accelerator.pad_across_processes(batch[\"labels\"], dim=1, pad_index=tokenizer.pad_token_id)\n\n                generated_tokens, labels = accelerator.gather_for_metrics((generated_tokens, labels))\n                generated_tokens = generated_tokens.cpu().numpy()\n                labels = labels.cpu().numpy()\n\n                if args.ignore_pad_token_for_loss:\n                    # Replace -100 in the labels as we can't decode them.\n                    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n                if isinstance(generated_tokens, tuple):\n                    generated_tokens = generated_tokens[0]\n                decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n                decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n                decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n                metric.add_batch(\n                    predictions=decoded_preds,\n                    references=decoded_labels,\n                )\n        result = metric.compute(use_stemmer=True)\n        result = {k: round(v * 100, 4) for k, v in result.items()}\n\n        logger.info(result)\n\n        if args.with_tracking:\n            result[\"train_loss\"] = total_loss.item() / len(train_dataloader)\n            result[\"epoch\"] = epoch\n            result[\"step\"] = completed_steps\n            accelerator.log(result, step=completed_steps)\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n            all_results = {f\"eval_{k}\": v for k, v in result.items()}\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n\n\nSUMMARY:\nThis code is the main training loop for a large language model. Here's a concise summary:\n\n**Overview**\n\nThe code trains a pre-trained language model on a custom dataset using transformers from Hugging Face's library. The training process involves two phases: inference and validation.\n\n**Key Components**\n\n1. **Training Loop**: This section contains the core training logic, including loading data, configuring hyperparameters, and iterating over batches to perform one step of backpropagation through time (PTT) or asynchronous training.\n2. **Inference Phase**: After a specified number of epochs, the model switches from training mode to inference mode. In this phase, the model evaluates itself on a validation set based on pre-defined evaluation metrics.\n3. **Evaluation and Results**: The final results are stored in various locations for sharing with others or saving locally.\n\n**Key Highlights**\n\n* The code assumes a custom dataset, tokenizer, and optimizer, which must be defined and configured beforehand.\n* The training process uses asynchronous optimization (PTT or async) to improve the efficiency of computation on multi-gpu systems.\n* The model is saved periodically during and after training in two locations: `args.output_dir` or in Hugging Face's model hub (`repo_id`).\n* Evaluation metrics are computed based on a post-processing function applied to both the predicted and reference token sequences.\n\n**Additional Notes**\n\nThe code likely relies heavily on the Hugging Face `transformers` library, which provides pre-trained models and efficient tools for NLP tasks. The extensive logging statements indicate that debugging is enabled.\n\nThis training script is likely part of a larger project or research paper, aiming to improve large language modeling capabilities by leveraging the best practices from state-of-the-art approaches in natural language processing (NLP)."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization.py:311-763

metadata:
{"docstring": "", "end_line": 763, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization.py", "name": "main", "start_line": 311, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_summarization\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    if data_args.source_prefix is None and model_args.model_name_or_path in [\n        \"google-t5/t5-small\",\n        \"google-t5/t5-base\",\n        \"google-t5/t5-large\",\n        \"google-t5/t5-3b\",\n        \"google-t5/t5-11b\",\n    ]:\n        logger.warning(\n            \"You're running a t5 model but didn't provide a source prefix, which is the expected, e.g. with \"\n            \"`--source_prefix 'summarize: ' `\"\n        )\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files this script will use the first column for the full texts and the second column for the\n    # summaries (unless you specify column names for this with the `text_column` and `summary_column` arguments).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        raw_datasets = load_dataset(\n            extension,\n            data_files=data_files,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        use_fast=model_args.use_fast_tokenizer,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    if model.config.decoder_start_token_id is None and isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):\n        if isinstance(tokenizer, MBartTokenizer):\n            model.config.decoder_start_token_id = tokenizer.lang_code_to_id[data_args.lang]\n        else:\n            model.config.decoder_start_token_id = tokenizer.convert_tokens_to_ids(data_args.lang)\n\n    if model.config.decoder_start_token_id is None:\n        raise ValueError(\"Make sure that `config.decoder_start_token_id` is correctly defined\")\n\n    if (\n        hasattr(model.config, \"max_position_embeddings\")\n        and model.config.max_position_embeddings < data_args.max_source_length\n    ):\n        if model_args.resize_position_embeddings is None:\n            logger.warning(\n                \"Increasing the model's number of position embedding vectors from\"\n                f\" {model.config.max_position_embeddings} to {data_args.max_source_length}.\"\n            )\n            model.resize_position_embeddings(data_args.max_source_length)\n        elif model_args.resize_position_embeddings:\n            model.resize_position_embeddings(data_args.max_source_length)\n        else:\n            raise ValueError(\n                f\"`--max_source_length` is set to {data_args.max_source_length}, but the model only has\"\n                f\" {model.config.max_position_embeddings} position encodings. Consider either reducing\"\n                f\" `--max_source_length` to {model.config.max_position_embeddings} or to automatically resize the\"\n                \" model's position encodings by passing `--resize_position_embeddings`.\"\n            )\n\n    prefix = data_args.source_prefix if data_args.source_prefix is not None else \"\"\n\n    # Preprocessing the datasets.\n    # We need to tokenize inputs and targets.\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        column_names = raw_datasets[\"train\"].column_names\n    elif training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        column_names = raw_datasets[\"validation\"].column_names\n    elif training_args.do_predict:\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        column_names = raw_datasets[\"test\"].column_names\n    else:\n        logger.info(\"There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.\")\n        return\n\n    if isinstance(tokenizer, tuple(MULTILINGUAL_TOKENIZERS)):\n        assert (\n            data_args.lang is not None\n        ), f\"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --lang argument\"\n\n        tokenizer.src_lang = data_args.lang\n        tokenizer.tgt_lang = data_args.lang\n\n        # For multilingual translation models like mBART-50 and M2M100 we need to force the target language token\n        # as the first generated token. We ask the user to explicitly provide this as --forced_bos_token argument.\n        forced_bos_token_id = (\n            tokenizer.lang_code_to_id[data_args.forced_bos_token] if data_args.forced_bos_token is not None else None\n        )\n        model.config.forced_bos_token_id = forced_bos_token_id\n\n    # Get the column names for input/target.\n    dataset_columns = summarization_name_mapping.get(data_args.dataset_name, None)\n    if data_args.text_column is None:\n        text_column = dataset_columns[0] if dataset_columns is not None else column_names[0]\n    else:\n        text_column = data_args.text_column\n        if text_column not in column_names:\n            raise ValueError(\n                f\"--text_column' value '{data_args.text_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n    if data_args.summary_column is None:\n        summary_column = dataset_columns[1] if dataset_columns is not None else column_names[1]\n    else:\n        summary_column = data_args.summary_column\n        if summary_column not in column_names:\n            raise ValueError(\n                f\"--summary_column' value '{data_args.summary_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n\n    # Temporarily set max_target_length for training.\n    max_target_length = data_args.max_target_length\n    padding = \"max_length\" if data_args.pad_to_max_length else False\n\n    if training_args.label_smoothing_factor > 0 and not hasattr(model, \"prepare_decoder_input_ids_from_labels\"):\n        logger.warning(\n            \"label_smoothing is enabled but the `prepare_decoder_input_ids_from_labels` method is not defined for \"\n            f\"`{model.__class__.__name__}`. This will lead to loss being calculated twice and will take up more memory\"\n        )\n\n    def preprocess_function(examples):\n        # remove pairs where at least one record is None\n\n        inputs, targets = [], []\n        for i in range(len(examples[text_column])):\n            if examples[text_column][i] and examples[summary_column][i]:\n                inputs.append(examples[text_column][i])\n                targets.append(examples[summary_column][i])\n\n        inputs = [prefix + inp for inp in inputs]\n        model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, padding=padding, truncation=True)\n\n        # Tokenize targets with the `text_target` keyword argument\n        labels = tokenizer(text_target=targets, max_length=max_target_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and data_args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    if training_args.do_train:\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n\n    if training_args.do_eval:\n        max_target_length = data_args.val_max_target_length\n        eval_dataset = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n\n    if training_args.do_predict:\n        max_target_length = data_args.val_max_target_length\n        predict_dataset = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n\n    # Data collator\n    label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n    data_collator = DataCollatorForSeq2Seq(\n        tokenizer,\n        model=model,\n        label_pad_token_id=label_pad_token_id,\n        pad_to_multiple_of=8 if training_args.fp16 else None,\n    )\n\n    # Metric\n    metric = evaluate.load(\"rouge\", cache_dir=model_args.cache_dir)\n\n    def postprocess_text(preds, labels):\n        preds = [pred.strip() for pred in preds]\n        labels = [label.strip() for label in labels]\n\n        # rougeLSum expects newline after each sentence\n        preds = [\"\\n\".join(nltk.sent_tokenize(pred)) for pred in preds]\n        labels = [\"\\n\".join(nltk.sent_tokenize(label)) for label in labels]\n\n        return preds, labels\n\n    def compute_metrics(eval_preds):\n        preds, labels = eval_preds\n        if isinstance(preds, tuple):\n            preds = preds[0]\n        # Replace -100s used for padding as we can't decode them\n        preds = np.where(preds != -100, preds, tokenizer.pad_token_id)\n        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n        # Some simple post-processing\n        decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n\n        result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n        result = {k: round(v * 100, 4) for k, v in result.items()}\n        prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n        result[\"gen_len\"] = np.mean(prediction_lens)\n        return result\n\n    # Override the decoding parameters of Seq2SeqTrainer\n    training_args.generation_max_length = (\n        training_args.generation_max_length\n        if training_args.generation_max_length is not None\n        else data_args.val_max_target_length\n    )\n    training_args.generation_num_beams = (\n        data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams\n    )\n\n    # Initialize our Trainer\n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics if training_args.predict_with_generate else None,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        metrics = train_result.metrics\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    results = {}\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n        if isinstance(eval_dataset, dict):\n            metrics = {}\n            for eval_ds_name, eval_ds in eval_dataset.items():\n                dataset_metrics = trainer.evaluate(eval_dataset=eval_ds, metric_key_prefix=f\"eval_{eval_ds_name}\")\n                metrics.update(dataset_metrics)\n        else:\n            metrics = trainer.evaluate(metric_key_prefix=\"eval\")\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n\n        predict_results = trainer.predict(predict_dataset, metric_key_prefix=\"predict\")\n        metrics = predict_results.metrics\n        max_predict_samples = (\n            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n        )\n        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n        if trainer.is_world_process_zero():\n            if training_args.predict_with_generate:\n                predictions = predict_results.predictions\n                predictions = np.where(predictions != -100, predictions, tokenizer.pad_token_id)\n                predictions = tokenizer.batch_decode(\n                    predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True\n                )\n                predictions = [pred.strip() for pred in predictions]\n                output_prediction_file = os.path.join(training_args.output_dir, \"generated_predictions.txt\")\n                with open(output_prediction_file, \"w\") as writer:\n                    writer.write(\"\\n\".join(predictions))\n\n    kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"summarization\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    if data_args.lang is not None:\n        kwargs[\"language\"] = data_args.lang\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n        trainer.create_model_card(**kwargs)\n\n\n\nSUMMARY:\nThis code is for training and testing a text summarization model using the Hugging Face library. Here's a high-level overview of what it does:\n\n1. **Data preparation**: Loads pre-processed data from a file ( dataset_path ) and splits it into training, validation, and prediction datasets.\n2. **Model configuration**: Configures the model with hyperparameters such as number of beams, max length for predictions, and other settings.\n3. **Trainer setup**: Creates a trainer instance to handle training, evaluation, and prediction tasks.\n4. **Training loop**: Trains the model using the `Seq2SeqTrainer` class, which uses Hugging Face's automatic mixed precision training and checkpointing mechanisms.\n5. **Evaluation and prediction**: Evaluates the model on validation datasets, measures its performance metrics (e.g., BLEU score), and makes predictions on test datasets.\n6. **Saving and uploading**: Saves the trained model in a specific directory (`output_dir`) and can be uploaded to a Hugging Face Hub.\n\nKey aspects of this code include:\n\n* Using Hugging Face's `Seq2SeqTrainer` class for efficient training and evaluation\n* Implementing custom metrics (e.g., BLEU score, prediction length) to evaluate model performance\n* Supporting various configuration options, such as language, dataset name, and training parameters"}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization.py:766-768

metadata:
{"docstring": "", "end_line": 768, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/summarization/run_summarization.py", "name": "_mp_fn", "start_line": 766, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Summary**\n\n`_mp_fn` is an internal function that calls the `main` function, likely for use with XLA Spawn (TPUs), but its context and purpose are unclear due to incomplete information. \n\n**Key Implementation Details:**\n\n* Calls the `main` function without specifying any additional parameters or transformations.\n\n**Notable Dependencies/Requirements:**\n\n* Uses XLA Spawn (TPUs) context manager, hinted by `_mp_fn`.\n* Reference to `main` function.\n* External dependency on XLA/TF inference framework."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_mlm_no_trainer.py:67-265

metadata:
{"docstring": "", "end_line": 265, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_mlm_no_trainer.py", "name": "parse_args", "start_line": 67, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a Masked Language Modeling task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--validation_split_percentage\",\n        default=5,\n        help=\"The percentage of the train set used as validation set in case there's no validation split\",\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\n        \"--max_seq_length\",\n        type=int,\n        default=None,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated.\"\n        ),\n    )\n    parser.add_argument(\n        \"--line_by_line\",\n        type=bool,\n        default=False,\n        help=\"Whether distinct lines of text in the dataset are to be handled as distinct sequences.\",\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\",\n        type=int,\n        default=None,\n        help=\"The number of processes to use for the preprocessing.\",\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--mlm_probability\", type=float, default=0.15, help=\"Ratio of tokens to mask for masked language modeling loss\"\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--low_cpu_mem_usage\",\n        action=\"store_true\",\n        help=(\n            \"It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. \"\n            \"If passed, LLM loading time and RAM consumption will be benefited.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.dataset_name is None and args.train_file is None and args.validation_file is None:\n        raise ValueError(\"Need either a dataset name or a training/validation file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            if extension not in [\"csv\", \"json\", \"txt\"]:\n                raise ValueError(\"`train_file` should be a csv, json or txt file.\")\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            if extension not in [\"csv\", \"json\", \"txt\"]:\n                raise ValueError(\"`validation_file` should be a csv, json or txt file.\")\n\n    if args.push_to_hub:\n        if args.output_dir is None:\n            raise ValueError(\"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\")\n\n\n\nSUMMARY:\n**Functionality Summary:**\n\nThe `parse_args()` function creates an ArgumentParser instance to parse command-line arguments for fine-tuning transformer models. It supports various hyperparameters, dataset configuration options, and model-specific parameters.\n\n**Key Implementation Details:**\n\n*   The parser includes arguments for dataset configuration (`dataset_name`, `.dataset_config_name`), training data files (`train_file`, `validation_file`), hyperparameter settings (e.g., batch size, learning rate), model architecture (model name or path, model type), and additional options for caching, logging, and model deployment.\n\n**Notable Dependencies:**\n\n*   The `argparse.ArgumentParser` class from the Python Standard Library\n*   External dependencies:\n    *   **_datasets**: for loading dataset files\n    *   **transformers**: for transformer models\n    *   **torch**: for PyTorch\n\n**Interfaces and Inherited Methods:**\n\n*   none explicitly stated, but inheritance from `argparse.ArgumentParser` implies the use of its methods (e.g., `parse_args`, `print_help`) to construct a parser that handles command-line arguments.\n\nHere is an example code snippet to demonstrate how this function might be used to create and parse command-line arguments for a transformer model fine-tuning scenario:\n\n```python\nimport argparse\n\n# Create the ArgumentParser instance, as shown in the provided function\nparser = argparse.ArgumentParser(description=\"Fine-tuning transformer models\")\n\n# Define some common hyperparameter settings\nparser.add_argument(\"--batch_size\", type=int, default=32)\nparser.add_argument(\"--learning_rate\", type=float, default=5e-4)\n\n# Define a few example dataset names and their paths\ndataset_config_names = [\"bert-base-uncased\"]\ndataset_paths = {\n    \"bert-base-uncased\": \"/data/bert-base-uncased\",\n}\n\ndef create_argparse_args():\n    # Parse command-line arguments, including the dataset configuration option, training data files, etc.\n    return parser.parse_args()\n\n# Create argument parses with specific configs\nargparse_obj = argparse.ArgumentParser()\nargparse_obj.add_argument(\"-dc\", \"--dataset-config-name\", type=str,\n                         default=dataset_config_names[0],\n                         choices=[name for name in dataset_config_names])\nargparse_obj.add_argument(\"-tr_file\", \"--train-file\", type=str, required=False)\nargparse_obj.add_argument(\"-va_file\", \"--validation-file\", type=str, required=False)\n... # Other args\n\n# Parse and return\nargs = create_argparse_args()\n\n```\n\nNote that you may choose your own options, as the provided `create_argparse_args()` method is just a demonstration of how similar such functions could be constructed using argparse. The structure should generally look like this:\n\n```python\n    parser.add_argument(name, type=int, default=32, help=\"Model architecture option\")\n```"}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_mlm_no_trainer.py:268-758

metadata:
{"docstring": "", "end_line": 758, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_mlm_no_trainer.py", "name": "main", "start_line": 268, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_mlm_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n        if \"validation\" not in raw_datasets.keys():\n            raw_datasets[\"validation\"] = load_dataset(\n                args.dataset_name,\n                args.dataset_config_name,\n                split=f\"train[:{args.validation_split_percentage}%]\",\n                trust_remote_code=args.trust_remote_code,\n            )\n            raw_datasets[\"train\"] = load_dataset(\n                args.dataset_name,\n                args.dataset_config_name,\n                split=f\"train[{args.validation_split_percentage}%:]\",\n                trust_remote_code=args.trust_remote_code,\n            )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        if extension == \"txt\":\n            extension = \"text\"\n        raw_datasets = load_dataset(extension, data_files=data_files)\n        # If no validation data is there, validation_split_percentage will be used to divide the dataset.\n        if \"validation\" not in raw_datasets.keys():\n            raw_datasets[\"validation\"] = load_dataset(\n                extension,\n                data_files=data_files,\n                split=f\"train[:{args.validation_split_percentage}%]\",\n            )\n            raw_datasets[\"train\"] = load_dataset(\n                extension,\n                data_files=data_files,\n                split=f\"train[{args.validation_split_percentage}%:]\",\n            )\n\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(args.config_name, trust_remote_code=args.trust_remote_code)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForMaskedLM.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            low_cpu_mem_usage=args.low_cpu_mem_usage,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForMaskedLM.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    column_names = raw_datasets[\"train\"].column_names\n    text_column_name = \"text\" if \"text\" in column_names else column_names[0]\n\n    if args.max_seq_length is None:\n        max_seq_length = tokenizer.model_max_length\n        if max_seq_length > 1024:\n            logger.warning(\n                \"The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value\"\n                \" of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can\"\n                \" override this default with `--block_size xxx`.\"\n            )\n            max_seq_length = 1024\n    else:\n        if args.max_seq_length > tokenizer.model_max_length:\n            logger.warning(\n                f\"The max_seq_length passed ({args.max_seq_length}) is larger than the maximum length for the \"\n                f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n            )\n        max_seq_length = min(args.max_seq_length, tokenizer.model_max_length)\n\n    if args.line_by_line:\n        # When using line_by_line, we just tokenize each nonempty line.\n        padding = \"max_length\" if args.pad_to_max_length else False\n\n        def tokenize_function(examples):\n            # Remove empty lines\n            examples[text_column_name] = [\n                line for line in examples[text_column_name] if len(line) > 0 and not line.isspace()\n            ]\n            return tokenizer(\n                examples[text_column_name],\n                padding=padding,\n                truncation=True,\n                max_length=max_seq_length,\n                # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it\n                # receives the `special_tokens_mask`.\n                return_special_tokens_mask=True,\n            )\n\n        with accelerator.main_process_first():\n            tokenized_datasets = raw_datasets.map(\n                tokenize_function,\n                batched=True,\n                num_proc=args.preprocessing_num_workers,\n                remove_columns=[text_column_name],\n                load_from_cache_file=not args.overwrite_cache,\n                desc=\"Running tokenizer on dataset line_by_line\",\n            )\n    else:\n        # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.\n        # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more\n        # efficient when it receives the `special_tokens_mask`.\n        def tokenize_function(examples):\n            return tokenizer(examples[text_column_name], return_special_tokens_mask=True)\n\n        with accelerator.main_process_first():\n            tokenized_datasets = raw_datasets.map(\n                tokenize_function,\n                batched=True,\n                num_proc=args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not args.overwrite_cache,\n                desc=\"Running tokenizer on every text in dataset\",\n            )\n\n        # Main data processing function that will concatenate all texts from our dataset and generate chunks of\n        # max_seq_length.\n        def group_texts(examples):\n            # Concatenate all texts.\n            concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}\n            total_length = len(concatenated_examples[list(examples.keys())[0]])\n            # We drop the small remainder, and if the total_length < max_seq_length  we exclude this batch and return an empty dict.\n            # We could add padding if the model supported it instead of this drop, you can customize this part to your needs.\n            total_length = (total_length // max_seq_length) * max_seq_length\n            # Split by chunks of max_len.\n            result = {\n                k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]\n                for k, t in concatenated_examples.items()\n            }\n            return result\n\n        # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a\n        # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value\n        # might be slower to preprocess.\n        #\n        # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:\n        # https://huggingface.co/docs/datasets/process#map\n\n        with accelerator.main_process_first():\n            tokenized_datasets = tokenized_datasets.map(\n                group_texts,\n                batched=True,\n                num_proc=args.preprocessing_num_workers,\n                load_from_cache_file=not args.overwrite_cache,\n                desc=f\"Grouping texts in chunks of {max_seq_length}\",\n            )\n\n    train_dataset = tokenized_datasets[\"train\"]\n    eval_dataset = tokenized_datasets[\"validation\"]\n\n    # Conditional for small test subsets\n    if len(train_dataset) > 3:\n        # Log a few random samples from the training set:\n        for index in random.sample(range(len(train_dataset)), 3):\n            logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # Data collator\n    # This one will take care of randomly masking the tokens.\n    data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=args.mlm_probability)\n\n    # DataLoaders creation:\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Note -> the training dataloader needs to be prepared before we grab his length below (cause its length will be\n    # shorter in multiprocess)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # On TPU, the tie weights in our model have been disconnected, so we need to restore the ties.\n    if accelerator.distributed_type == DistributedType.TPU:\n        model.tie_weights()\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"mlm_no_trainer\", experiment_config)\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n        losses = []\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n\n            loss = outputs.loss\n            losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))\n\n        losses = torch.cat(losses)\n        try:\n            eval_loss = torch.mean(losses)\n            perplexity = math.exp(eval_loss)\n        except OverflowError:\n            perplexity = float(\"inf\")\n\n        logger.info(f\"epoch {epoch}: perplexity: {perplexity} eval_loss: {eval_loss}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"perplexity\": perplexity,\n                    \"eval_loss\": eval_loss,\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n\n\nSUMMARY:\nHere is a concise summary of the code:\n\n** overview**\n\nThis is a training script for a language model using the Hugging Face Transformers library. It trains the model on a dataset and evaluates its performance.\n\n**main components**\n\n1. **Data loading**: The script loads the dataset and shuffles it.\n2. **Model initialization**: The script initializes the model with the specified configuration and architecture.\n3. **Training loop**: The script runs through a series of epochs, where:\n\t* It trains the model on the shuffled data for a specified number of steps.\n\t* It evaluates the model's performance using evaluation metrics (e.g., perplexity).\n\t* It saves the model checkpoints at regular intervals.\n4. **Tracking and logging**: The scripttracks progress, logs metrics, and saves results to a file and/or upload them to Model Hub (if `push_to_hub` is enabled).\n\n**Details**\n\nThe script uses the following libraries:\n\n* Hugging Face Transformers\n* Torch\n\nIt leverages the Hugging Face framework for efficient training and evaluation, including utilities such as:\n\n* Auto-optimization of hyperparameters\n* Automatic logging and tracking of metrics\n* Saving and uploading models to Model Hub (if `push_to_hub` is enabled)\n\n**Notes**\n\nThe script assumes that you have installed the required libraries and have a valid hub token for pushing models to Model Hub. The output directory (`output_dir`) specified in the script should be created before running the script, as it will store the model checkpoints and evaluation results."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_fim_no_trainer.py:72-320

metadata:
{"docstring": "", "end_line": 320, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_fim_no_trainer.py", "name": "parse_args", "start_line": 72, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(\n        description=\"Finetune a transformers model on a causal language modeling task using fill-in-the middle objective\"\n    )\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv, txt or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv, txt or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--validation_split_percentage\",\n        default=5,\n        help=\"The percentage of the train set used as validation set in case there's no validation split\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=42, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\n        \"--block_size\",\n        type=int,\n        default=None,\n        help=(\n            \"Optional input sequence length after tokenization. The training dataset will be truncated in block of\"\n            \" this size for training. Default to the model max input length for single sentence inputs (take into\"\n            \" account special tokens).\"\n        ),\n    )\n    parser.add_argument(\n        \"--fim_rate\",\n        type=float,\n        default=0.5,\n        help=(\n            \" Optional probability with which the FIM transformation is applied to the example.\"\n            \" Default is 0.5. A rate of 1.0 means every example will undergo FIM transformation,\"\n            \" while a rate of 0.0 means no example will.\"\n        ),\n    )\n    parser.add_argument(\n        \"--fim_spm_rate\",\n        type=float,\n        default=0.5,\n        help=(\n            \"Within the examples undergoing FIM transformation, this rate determines the probability\"\n            \" of applying the Sentence Permutation Mode (SPM).\"\n            \" Default is 0.5. A rate of 1.0 means all FIM transformations will use SPM,\"\n            \" while a rate of 0.0 means none will.\"\n        ),\n    )\n    parser.add_argument(\n        \"--truncate_or_pad\",\n        type=bool,\n        default=True,\n        help=(\n            \"Indicates whether the transformed example should be truncated or padded to maintain\"\n            \" the same length as the original example.\"\n            \" Default is True. If False, the function will not truncate or pad the examples.\"\n        ),\n    )\n    parser.add_argument(\n        \"--fim_prefix_token\",\n        type=str,\n        default=\"<fim_prefix>\",\n        help=\"Fill-in-Middle Prefix token. Defaults to '<fim_prefix>'.\",\n    )\n    parser.add_argument(\n        \"--fim_middle_token\",\n        type=str,\n        default=\"<fim_middle>\",\n        help=\"Fill-in-Middle Middle token. Defaults to '<fim_middle>'.\",\n    )\n    parser.add_argument(\n        \"--fim_suffix_token\",\n        type=str,\n        default=\"<fim_suffix>\",\n        help=\"Fill-in-Middle Middle token. Defaults to '<fim_suffix>'.\",\n    )\n    parser.add_argument(\n        \"--fim_pad_token\",\n        type=str,\n        default=\"<fim_pad>\",\n        help=(\n            \"Fill-in-Middle Pad token. Used only when 'truncate_or_pad' is set to True.\" \" Defaults to '<fim_pad>'.\"\n        ),\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\",\n        type=int,\n        default=None,\n        help=\"The number of processes to use for the preprocessing.\",\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--no_keep_linebreaks\", action=\"store_true\", help=\"Do not keep line breaks when using TXT files.\"\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--low_cpu_mem_usage\",\n        action=\"store_true\",\n        help=(\n            \"It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. \"\n            \"If passed, LLM loading time and RAM consumption will be benefited.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.dataset_name is None and args.train_file is None and args.validation_file is None:\n        raise ValueError(\"Need either a dataset name or a training/validation file.\")\n    else:\n        if args.train_file is not None:\n            extension = args.train_file.split(\".\")[-1]\n            if extension not in [\"csv\", \"json\", \"txt\"]:\n                raise ValueError(\"`train_file` should be a csv, json or txt file.\")\n        if args.validation_file is not None:\n            extension = args.validation_file.split(\".\")[-1]\n            if extension not in [\"csv\", \"json\", \"txt\"]:\n                raise ValueError(\"`validation_file` should be a csv, json or txt file.\")\n\n    if args.push_to_hub:\n        if args.output_dir is None:\n            raise ValueError(\"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\")\n\n\n\nSUMMARY:\nThe provided code snippet is written in Python and appears to be the main entry point for a script that utilizes the Hugging Face Transformers library for natural language processing tasks. The script accepts various command-line arguments that define how the model should be trained, saved, and potentially pushed to the Model Hub.\n\nHere's a summary:\n\n*   **Usage**: This script can be run from the command line, passing in various arguments that control its behavior.\n*   **Arguments**:\n    *   Training and validation file paths (csv, json, or txt format)\n    *   Hyperparameters for training, such as learning rate, batch size, etc.\n    *   Model configuration options\n    *   Integration with the Model Hub\n    *   Experiment tracking options\n\n**Example Argument Parsers:**\n\n```python\n# Example argument parsers\nif __name__ == \"__main__\":\n    parser = TrainerArguments(\n        resume_model=None,\n        output_dir=None,\n        push_to_hub=False,\n        hub_model_id=None,\n        hub_token=None,\n        save_steps=700,\n        save_total_limit=None,\n        save_on_each_device=True,\n        num_train_epochs=3.0,\n        per_device train_batch_size=16,\n        per_device_val_batch_size=64,\n        warmup_steps=500,\n        weight_decay=0.01,  # Add this argument\n    )\n```\n\n**Key Features:**\n\n*   **Training**: Automatic model training and tuning.\n*   **Model Saving**: Model saving options, including saving on each device or total limit.\n*   **Hub Integration**: Push the trained model to the Model Hub for sharing and collaboration."}


created_at: 2025-02-05 09:13:39
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_fim_no_trainer.py:323-911

metadata:
{"docstring": "", "end_line": 911, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/language-modeling/run_fim_no_trainer.py", "name": "main", "start_line": 323, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_fim_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n        # Set a numpy random state for FIM transformations\n        np_rng = np.random.RandomState(seed=args.seed)\n    else:\n        # Still set a random state for FIM transformations\n        np_rng = np.random.RandomState(seed=42)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            repo_id = create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n            # Clone repo locally\n            repo = Repository(args.output_dir, clone_from=repo_id, token=args.hub_token)\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n        if \"validation\" not in raw_datasets.keys():\n            raw_datasets[\"validation\"] = load_dataset(\n                args.dataset_name,\n                args.dataset_config_name,\n                split=f\"train[:{args.validation_split_percentage}%]\",\n                trust_remote_code=args.trust_remote_code,\n            )\n            raw_datasets[\"train\"] = load_dataset(\n                args.dataset_name,\n                args.dataset_config_name,\n                split=f\"train[{args.validation_split_percentage}%:]\",\n                trust_remote_code=args.trust_remote_code,\n            )\n    else:\n        data_files = {}\n        dataset_args = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n        extension = args.train_file.split(\".\")[-1]\n        if extension == \"txt\":\n            extension = \"text\"\n            dataset_args[\"keep_linebreaks\"] = not args.no_keep_linebreaks\n        raw_datasets = load_dataset(extension, data_files=data_files, **dataset_args)\n        # If no validation data is there, validation_split_percentage will be used to divide the dataset.\n        if \"validation\" not in raw_datasets.keys():\n            raw_datasets[\"validation\"] = load_dataset(\n                extension,\n                data_files=data_files,\n                split=f\"train[:{args.validation_split_percentage}%]\",\n                **dataset_args,\n            )\n            raw_datasets[\"train\"] = load_dataset(\n                extension,\n                data_files=data_files,\n                split=f\"train[{args.validation_split_percentage}%:]\",\n                **dataset_args,\n            )\n\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.html.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(\n            args.config_name,\n            trust_remote_code=args.trust_remote_code,\n        )\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(\n            args.model_name_or_path,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForCausalLM.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            low_cpu_mem_usage=args.low_cpu_mem_usage,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForCausalLM.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # Add the new FIM tokens to the tokenizer and resize model's vocab embeddings\n    special_tokens = [args.fim_prefix_token, args.fim_middle_token, args.fim_suffix_token]\n    if args.truncate_or_pad:\n        special_tokens.append(args.fim_pad_token)\n\n    # Get the factor by which the embedding layer should be padded based on the device\n    pad_factor = 1\n    if torch.cuda.is_availble():\n        pad_factor = 8\n\n    elif is_torch_tpu_available():\n        pad_factor = 128\n\n    # Add the new tokens to the tokenizer\n    tokenizer.add_tokens(special_tokens)\n    original_embeddings = model.get_input_embeddings()\n\n    if is_deepspeed_zero3_enabled():\n        import deepspeed\n\n        with deepspeed.zero.GatheredParameters(original_embeddings.weight, modifier_rank=0):\n            # Get the pre-expansion embeddings of the model and resize the embedding layer\n            model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=pad_factor)\n            embeddings = model.get_input_embeddings()\n\n            # Sample the embeddings for the new tokens from a multivariate normal distribution\n            # We do this so that the new embeddings are close to the original embeddings and not necessarily zero\n            # More on this: https://nlp.stanford.edu/~johnhew/vocab-expansion.html\n            mean = original_embeddings.mean(dim=0)\n            n = original_embeddings.size()[0]\n            sigma = ((original_embeddings - mean).T @ (original_embeddings - mean)) / n\n            dist = torch.distributions.multivariate_normal.MultivariateNormal(\n                mean,\n                covariance_matrix=1e-5 * sigma,\n            )\n            new_token_embeddings = torch.stack(\n                tuple((dist.sample() for _ in range(len(special_tokens)))),\n                dim=0,\n            )\n    else:\n        original_embeddings = model.get_input_embeddings()\n        # Get the pre-expansion embeddings of the model and resize the embedding layer\n        model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=pad_factor)\n        embeddings = model.get_input_embeddings()\n\n        # Sample the embeddings for the new tokens from a multivariate normal distribution\n        # We do this so that the new embeddings are close to the original embeddings and not necessarily zero\n        # More on this: https://nlp.stanford.edu/~johnhew/vocab-expansion.html\n        mean = original_embeddings.mean(dim=0)\n        n = original_embeddings.size()[0]\n        sigma = ((original_embeddings - mean).T @ (original_embeddings - mean)) / n\n        dist = torch.distributions.multivariate_normal.MultivariateNormal(\n            mean,\n            covariance_matrix=1e-5 * sigma,\n        )\n        new_token_embeddings = torch.stack(\n            tuple((dist.sample() for _ in range(len(special_tokens)))),\n            dim=0,\n        )\n\n    if is_deepspeed_zero3_enabled():\n        import deepspeed\n\n        with deepspeed.zero.GatheredParameters(embeddings.weight, modifier_rank=0):\n            # Set the new tokens' embeddings to the newly sampled embeddings\n            embeddings.weight.data[-len(special_tokens) :] = new_token_embeddings\n    else:\n        # Set the new tokens' embeddings to the newly sampled embeddings\n        embeddings.weight.data[-len(special_tokens) :] = new_token_embeddings\n\n    # Update the model's embeddings with the new embeddings\n    model.set_input_embeddings(embeddings)\n\n    logger.info(\"Added special tokens to the tokenizer and resized model's embedding layer\")\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    column_names = raw_datasets[\"train\"].column_names\n    text_column_name = \"text\" if \"text\" in column_names else column_names[0]\n\n    def tokenize_function(examples):\n        return tokenizer(examples[text_column_name])\n\n    with accelerator.main_process_first():\n        tokenized_datasets = raw_datasets.map(\n            tokenize_function,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on dataset\",\n        )\n\n    if args.block_size is None:\n        block_size = tokenizer.model_max_length\n        if block_size > config.max_position_embeddings:\n            logger.warning(\n                f\"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). \"\n                f\"Using block_size={min(1024, config.max_position_embeddings)} instead. You can change that default value by passing --block_size xxx.\"\n            )\n            block_size = min(1024, config.max_position_embeddings)\n    else:\n        if args.block_size > tokenizer.model_max_length:\n            logger.warning(\n                f\"The block_size passed ({args.block_size}) is larger than the maximum length for the model \"\n                f\"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}.\"\n            )\n        block_size = min(args.block_size, tokenizer.model_max_length)\n\n    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.\n    def group_texts(examples):\n        # Concatenate all texts.\n        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}\n        total_length = len(concatenated_examples[list(examples.keys())[0]])\n        # We drop the small remainder, and if the total_length < block_size  we exclude this batch and return an empty dict.\n        # We could add padding if the model supported it instead of this drop, you can customize this part to your needs.\n        total_length = (total_length // block_size) * block_size\n        # Split by chunks of max_len.\n        result = {\n            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n            for k, t in concatenated_examples.items()\n        }\n        result[\"labels\"] = result[\"input_ids\"].copy()\n        return result\n\n    # Get the FIM-specific token ids\n    prefix_tok_id = tokenizer.convert_tokens_to_ids(args.fim_prefix_token)\n    middle_tok_id = tokenizer.convert_tokens_to_ids(args.fim_middle_token)\n    suffix_tok_id = tokenizer.convert_tokens_to_ids(args.fim_suffix_token)\n    pad_tok_id = None\n\n    # If truncate_or_pad is on, also get pad token id\n    if args.truncate_or_pad:\n        pad_tok_id = tokenizer.convert_tokens_to_ids(args.fim_pad_token)\n\n    # The two functions below perform the FIM transformation on the data (either PSM or SPM or PSM+SPM)\n    # Don't call fim_transform directly in .map()\n    # Adapted from https://github.com/loubnabnl/santacoder-finetuning/blob/main/fim.py#L22C13-L83\n    def fim_transform(example):\n        \"\"\"\n        This function performs FIM transformation on a single example (list of tokens)\n        \"\"\"\n        if np_rng.binomial(1, args.fim_rate):\n            boundaries = sorted(np_rng.randint(low=0, high=len(example) + 1, size=2))\n\n            prefix = example[: boundaries[0]]\n            middle = example[boundaries[0] : boundaries[1]]\n            suffix = example[boundaries[1] :]\n\n            if args.truncate_or_pad:\n                total_length = len(prefix) + len(middle) + len(suffix) + 3\n                diff = total_length - len(example)\n                if diff > 0:\n                    suffix = suffix[: max(0, len(suffix) - diff)]\n                elif diff < 0:\n                    suffix.extend([pad_tok_id] * (-diff))\n\n            if np_rng.binomial(1, args.fim_spm_rate):\n                # Apply Suffix-Prefix-Middle (SPM) transformation\n                transformed_example = [prefix_tok_id, suffix_tok_id] + suffix + [middle_tok_id] + prefix + middle\n            else:\n                # Apply Prefix-Suffix-Middle (PSM) transformation\n                transformed_example = [prefix_tok_id] + prefix + [suffix_tok_id] + suffix + [middle_tok_id] + middle\n        else:\n            transformed_example = example\n\n        return transformed_example\n\n    # Below function is the one you are supposed to call in the .map() function\n    def apply_fim(examples):\n        \"\"\"\n        Apply FIM transformation to a batch of examples\n        \"\"\"\n        fim_transform_ids = [fim_transform(ids) for ids in examples[\"input_ids\"]]\n        examples[\"input_ids\"] = fim_transform_ids\n        examples[\"labels\"] = fim_transform_ids\n        # If your application requires custom attention mask, please adjust this function's below line.\n        # Since FIM transformation increases the number of tokens in input_ids and labels\n        # but leaves the number of tokens unchanged in attention_masks which would cause problems\n        examples[\"attention_mask\"] = [[1] * len(mask) for mask in examples[\"input_ids\"]]\n        return examples\n\n    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder\n    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower\n    # to preprocess.\n    #\n    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:\n    # https://huggingface.co/docs/datasets/process#map\n\n    # FIM transformations are only supposed to be applied before group_texts processing otherwise some sentences will\n    # have 3-4 more tokens than others due to probabilistic addition of FIM-specific tokens which will raise errors\n    with accelerator.main_process_first():\n        fim_datasets = tokenized_datasets.map(\n            apply_fim,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Performing FIM transformation\",\n        )\n        lm_datasets = fim_datasets.map(\n            group_texts,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=f\"Grouping texts in chunks of {block_size}\",\n        )\n\n    train_dataset = lm_datasets[\"train\"]\n    eval_dataset = lm_datasets[\"validation\"]\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(\n        eval_dataset, collate_fn=default_data_collator, batch_size=args.per_device_eval_batch_size\n    )\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"layer_norm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * args.gradient_accumulation_steps,\n        num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # On TPU, the tie weights in our model have been disconnected, so we need to restore the ties.\n    if accelerator.distributed_type == DistributedType.TPU:\n        model.tie_weights()\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"fim_no_trainer\", experiment_config)\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n        losses = []\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n\n            loss = outputs.loss\n            losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))\n\n        losses = torch.cat(losses)\n        try:\n            eval_loss = torch.mean(losses)\n            perplexity = math.exp(eval_loss)\n        except OverflowError:\n            perplexity = float(\"inf\")\n\n        logger.info(f\"epoch {epoch}: perplexity: {perplexity} eval_loss: {eval_loss}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"perplexity\": perplexity,\n                    \"eval_loss\": eval_loss,\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                repo.push_to_hub(\n                    commit_message=f\"Training in progress epoch {epoch}\", blocking=False, auto_lfs_prune=True\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                repo.push_to_hub(commit_message=\"End of training\", auto_lfs_prune=True)\n\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n\n\nSUMMARY:\nHere is a concise summary:\n\nThis is the code for a PyTorch-based machine translation model training script. The script trains the model on an evaluation dataset, keeps track of losses and perplexities at each epoch, and saves the final model to a specified output directory when training completes. The script also has the option to push the final model to a Hugging Face Model Hub repository and display progress updates during training.\n\nKey features include:\n\n* Training on an evaluation dataset with batch-sized inference\n* Tracking losses and perplexities at each epoch\n* Saving the final model to a specified output directory\n* Option to push the final model to a Hugging Face Model Hub repository\n* Progress updates during training\n\nNote that some parts of the code are commented out or not clear, which may require further clarification or debugging."}


