# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: sysdb
filename: 00008-maintenance-log.sqlite.sql
hash: 0a0e7e93111a01789addf64961c6127c
sql:
-- Records when database maintenance operations are performed.
-- At time of creation, this table is only used to record vacuum operations.
CREATE TABLE maintenance_log (
  id INT PRIMARY KEY,
  timestamp INT NOT NULL,
  operation TEXT NOT NULL
);


dir: sysdb
filename: 00009-segment-collection-not-null.sqlite.sql
hash: 054355aef9e63702bf54ea29e61563f1
sql:
-- This makes segments.collection non-nullable.
CREATE TABLE segments_temp (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    collection TEXT REFERENCES collection(id) NOT NULL
);

INSERT INTO segments_temp SELECT * FROM segments;
DROP TABLE segments;
ALTER TABLE segments_temp RENAME TO segments;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


dir: metadb
filename: 00004-metadata-indices.sqlite.sql
hash: fb36603a45ee2cd0254cef3ef86585e8
sql:
CREATE INDEX IF NOT EXISTS embedding_metadata_int_value ON embedding_metadata (key, int_value) WHERE int_value IS NOT NULL;
CREATE INDEX IF NOT EXISTS embedding_metadata_float_value ON embedding_metadata (key, float_value) WHERE float_value IS NOT NULL;
CREATE INDEX IF NOT EXISTS embedding_metadata_string_value ON embedding_metadata (key, string_value) WHERE string_value IS NOT NULL;


# embeddings_queue

created_at: 2025-02-05 09:13:35
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim_no_trainer.py:375-799

metadata:
{"docstring": "", "end_line": 799, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim_no_trainer.py", "name": "main", "start_line": 375, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_mim_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(\n        gradient_accumulation_steps=args.gradient_accumulation_steps,\n        **accelerator_log_kwargs,\n    )\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Initialize our dataset.\n    ds = load_dataset(\n        args.dataset_name,\n        args.dataset_config_name,\n        data_files=args.data_files,\n        cache_dir=args.cache_dir,\n        token=args.token,\n        trust_remote_code=args.trust_remote_code,\n    )\n\n    # If we don't have a validation split, split off a percentage of train as validation.\n    args.train_val_split = None if \"validation\" in ds.keys() else args.train_val_split\n    if isinstance(args.train_val_split, float) and args.train_val_split > 0.0:\n        split = ds[\"train\"].train_test_split(args.train_val_split)\n        ds[\"train\"] = split[\"train\"]\n        ds[\"validation\"] = split[\"test\"]\n\n    # Create config\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config_kwargs = {\n        \"cache_dir\": args.cache_dir,\n        \"revision\": args.model_revision,\n        \"token\": args.token,\n        \"trust_remote_code\": args.trust_remote_code,\n    }\n    if args.config_name_or_path:\n        config = AutoConfig.from_pretrained(args.config_name_or_path, **config_kwargs)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, **config_kwargs)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n        if args.config_overrides is not None:\n            logger.info(f\"Overriding config: {args.config_overrides}\")\n            config.update_from_string(args.config_overrides)\n            logger.info(f\"New config: {config}\")\n\n    # make sure the decoder_type is \"simmim\" (only relevant for BEiT)\n    if hasattr(config, \"decoder_type\"):\n        config.decoder_type = \"simmim\"\n\n    # adapt config\n    args.image_size = args.image_size if args.image_size is not None else config.image_size\n    args.patch_size = args.patch_size if args.patch_size is not None else config.patch_size\n    args.encoder_stride = args.encoder_stride if args.encoder_stride is not None else config.encoder_stride\n\n    config.update(\n        {\n            \"image_size\": args.image_size,\n            \"patch_size\": args.patch_size,\n            \"encoder_stride\": args.encoder_stride,\n        }\n    )\n\n    # create image processor\n    if args.image_processor_name:\n        image_processor = AutoImageProcessor.from_pretrained(args.image_processor_name, **config_kwargs)\n    elif args.model_name_or_path:\n        image_processor = AutoImageProcessor.from_pretrained(args.model_name_or_path, **config_kwargs)\n    else:\n        IMAGE_PROCESSOR_TYPES = {\n            conf.model_type: image_processor_class for conf, image_processor_class in IMAGE_PROCESSOR_MAPPING.items()\n        }\n        image_processor = IMAGE_PROCESSOR_TYPES[args.model_type]()\n\n    # create model\n    if args.model_name_or_path:\n        model = AutoModelForMaskedImageModeling.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            cache_dir=args.cache_dir,\n            revision=args.model_revision,\n            token=args.token,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForMaskedImageModeling.from_config(\n            config,\n            token=args.token,\n            trust_remote_code=args.trust_remote_code,\n        )\n\n    column_names = ds[\"train\"].column_names\n\n    if args.image_column_name is not None:\n        image_column_name = args.image_column_name\n    elif \"image\" in column_names:\n        image_column_name = \"image\"\n    elif \"img\" in column_names:\n        image_column_name = \"img\"\n    else:\n        image_column_name = column_names[0]\n\n    # transformations as done in original SimMIM paper\n    # source: https://github.com/microsoft/SimMIM/blob/main/data/data_simmim.py\n    transforms = Compose(\n        [\n            Lambda(lambda img: img.convert(\"RGB\")),\n            RandomResizedCrop(args.image_size, scale=(0.67, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)),\n            RandomHorizontalFlip(),\n            ToTensor(),\n            Normalize(mean=image_processor.image_mean, std=image_processor.image_std),\n        ]\n    )\n\n    # create mask generator\n    mask_generator = MaskGenerator(\n        input_size=args.image_size,\n        mask_patch_size=args.mask_patch_size,\n        model_patch_size=args.patch_size,\n        mask_ratio=args.mask_ratio,\n    )\n\n    def preprocess_images(examples):\n        \"\"\"Preprocess a batch of images by applying transforms + creating a corresponding mask, indicating\n        which patches to mask.\"\"\"\n\n        examples[\"pixel_values\"] = [transforms(image) for image in examples[image_column_name]]\n        examples[\"mask\"] = [mask_generator() for i in range(len(examples[image_column_name]))]\n\n        return examples\n\n    if args.max_train_samples is not None:\n        ds[\"train\"] = ds[\"train\"].shuffle(seed=args.seed).select(range(args.max_train_samples))\n    # Set the training transforms\n    ds[\"train\"].set_transform(preprocess_images)\n\n    if args.max_eval_samples is not None:\n        ds[\"validation\"] = ds[\"validation\"].shuffle(seed=args.seed).select(range(args.max_eval_samples))\n    # Set the validation transforms\n    ds[\"validation\"].set_transform(preprocess_images)\n\n    # DataLoaders creation:\n    train_dataloader = DataLoader(\n        ds[\"train\"],\n        shuffle=True,\n        collate_fn=collate_fn,\n        batch_size=args.per_device_train_batch_size,\n    )\n    eval_dataloader = DataLoader(\n        ds[\"validation\"],\n        collate_fn=collate_fn,\n        batch_size=args.per_device_eval_batch_size,\n    )\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Note -> the training dataloader needs to be prepared before we grab his length below (cause its length will be\n    # shorter in multiprocess)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model,\n        optimizer,\n        train_dataloader,\n        eval_dataloader,\n        lr_scheduler,\n    )\n\n    # On TPU, the tie weights in our model have been disconnected, so we need to restore the ties.\n    if accelerator.distributed_type == DistributedType.TPU:\n        model.tie_weights()\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"mim_no_trainer\", experiment_config)\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(ds['train'])}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(int(args.max_train_steps)), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n        losses = []\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n\n            loss = outputs.loss\n            losses.append(accelerator.gather_for_metrics(loss.repeat(args.per_device_eval_batch_size)))\n\n        losses = torch.cat(losses)\n        eval_loss = torch.mean(losses)\n\n        logger.info(f\"epoch {epoch}: eval_loss: {eval_loss}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"eval_loss\": eval_loss,\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                image_processor.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            image_processor.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n\n\nSUMMARY:\nThis code is a template for a PyTorch-based reinforcement learning framework, likely designed to be used with the Hugging Face library for model fine-tuning and deployment. It consists of training loop that iterates over an evaluation dataset, calculating the loss and tracking metrics.\n\nHere's a concise summary:\n\n**Training Loop:**\n\n1. For each epoch:\n\t* Update the model parameters (`model.train()`).\n\t* Create a dataloader (`active_dataloader`) from the training data.\n\t* Iterate over the batches in the dataloader.\n\t\t+ Accumulate gradients using `torch.distributions`\n\t\t+ Update the model parameters with `optimizer.step()`\n\t\t+ Update the learning rate schedule with `lr_scheduler.step()`.\n2. After each epoch, evaluate the model on a validation dataset (`model.eval()`).\n3. Track metrics such as loss and training accuracy during evaluation.\n\n**Additional Components:**\n\n* Tracking evaluation metrics using Hugging Face's tracking system.\n* Saving model checkpoints at regular intervals (e.g., every `checkpointing_steps` epochs) for fine-tuning and deployment.\n\nOverall, this code provides a basic framework for training a PyTorch model on an RL or reinforcement learning task, with optional components for tracking performance and saving models."}


created_at: 2025-02-05 09:13:35
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py:169-171
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 171, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py", "name": "collate_fn", "start_line": 169, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(examples):\n    pixel_values = torch.stack([example[\"pixel_values\"] for example in examples])\n\n\nSUMMARY:\n**Summary:**\n\n`collate_fn` is a function that collates input data (`examples`) into a standardized format. It stacks the \"pixel_values\" from each `example` and returns a dictionary with the collated values, where the key is `\"pixel_values\"`.\n\n**Key Implementation Details:**\n\n* Uses PyTorch's `torch.stack` to combine pixel values.\n* Outputs a dictionary containing a single key-value pair: `\"pixel_values\" -> stacked tensor of pixel values.\n\n**Note:**\n\nThis function appears to be part of a larger data processing pipeline, likely used in deep learning applications."}


created_at: 2025-02-05 09:13:35
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py:174-398
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 398, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py", "name": "main", "start_line": 174, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, CustomTrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_mae\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Initialize our dataset.\n    ds = load_dataset(\n        data_args.dataset_name,\n        data_args.dataset_config_name,\n        data_files=data_args.data_files,\n        cache_dir=model_args.cache_dir,\n        token=model_args.token,\n        trust_remote_code=data_args.trust_remote_code,\n    )\n\n    # If we don't have a validation split, split off a percentage of train as validation.\n    data_args.train_val_split = None if \"validation\" in ds.keys() else data_args.train_val_split\n    if isinstance(data_args.train_val_split, float) and data_args.train_val_split > 0.0:\n        split = ds[\"train\"].train_test_split(data_args.train_val_split)\n        ds[\"train\"] = split[\"train\"]\n        ds[\"validation\"] = split[\"test\"]\n\n    # Load pretrained model and image processor\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config_kwargs = {\n        \"cache_dir\": model_args.cache_dir,\n        \"revision\": model_args.model_revision,\n        \"token\": model_args.token,\n    }\n    if model_args.config_name:\n        config = ViTMAEConfig.from_pretrained(model_args.config_name, **config_kwargs)\n    elif model_args.model_name_or_path:\n        config = ViTMAEConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n    else:\n        config = ViTMAEConfig()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n        if model_args.config_overrides is not None:\n            logger.info(f\"Overriding config: {model_args.config_overrides}\")\n            config.update_from_string(model_args.config_overrides)\n            logger.info(f\"New config: {config}\")\n\n    # adapt config\n    config.update(\n        {\n            \"mask_ratio\": model_args.mask_ratio,\n            \"norm_pix_loss\": model_args.norm_pix_loss,\n        }\n    )\n\n    # create image processor\n    if model_args.image_processor_name:\n        image_processor = ViTImageProcessor.from_pretrained(model_args.image_processor_name, **config_kwargs)\n    elif model_args.model_name_or_path:\n        image_processor = ViTImageProcessor.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n    else:\n        image_processor = ViTImageProcessor()\n\n    # create model\n    if model_args.model_name_or_path:\n        model = ViTMAEForPreTraining.from_pretrained(\n            model_args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n            config=config,\n            cache_dir=model_args.cache_dir,\n            revision=model_args.model_revision,\n            token=model_args.token,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = ViTMAEForPreTraining(config)\n\n    if training_args.do_train:\n        column_names = ds[\"train\"].column_names\n    else:\n        column_names = ds[\"validation\"].column_names\n\n    if data_args.image_column_name is not None:\n        image_column_name = data_args.image_column_name\n    elif \"image\" in column_names:\n        image_column_name = \"image\"\n    elif \"img\" in column_names:\n        image_column_name = \"img\"\n    else:\n        image_column_name = column_names[0]\n\n    # transformations as done in original MAE paper\n    # source: https://github.com/facebookresearch/mae/blob/main/main_pretrain.py\n    if \"shortest_edge\" in image_processor.size:\n        size = image_processor.size[\"shortest_edge\"]\n    else:\n        size = (image_processor.size[\"height\"], image_processor.size[\"width\"])\n    transforms = Compose(\n        [\n            Lambda(lambda img: img.convert(\"RGB\") if img.mode != \"RGB\" else img),\n            RandomResizedCrop(size, scale=(0.2, 1.0), interpolation=InterpolationMode.BICUBIC),\n            RandomHorizontalFlip(),\n            ToTensor(),\n            Normalize(mean=image_processor.image_mean, std=image_processor.image_std),\n        ]\n    )\n\n    def preprocess_images(examples):\n        \"\"\"Preprocess a batch of images by applying transforms.\"\"\"\n\n        examples[\"pixel_values\"] = [transforms(image) for image in examples[image_column_name]]\n        return examples\n\n    if training_args.do_train:\n        if \"train\" not in ds:\n            raise ValueError(\"--do_train requires a train dataset\")\n        if data_args.max_train_samples is not None:\n            ds[\"train\"] = ds[\"train\"].shuffle(seed=training_args.seed).select(range(data_args.max_train_samples))\n        # Set the training transforms\n        ds[\"train\"].set_transform(preprocess_images)\n\n    if training_args.do_eval:\n        if \"validation\" not in ds:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        if data_args.max_eval_samples is not None:\n            ds[\"validation\"] = (\n                ds[\"validation\"].shuffle(seed=training_args.seed).select(range(data_args.max_eval_samples))\n            )\n        # Set the validation transforms\n        ds[\"validation\"].set_transform(preprocess_images)\n\n    # Compute absolute learning rate\n    total_train_batch_size = (\n        training_args.train_batch_size * training_args.gradient_accumulation_steps * training_args.world_size\n    )\n    if training_args.base_learning_rate is not None:\n        training_args.learning_rate = training_args.base_learning_rate * total_train_batch_size / 256\n\n    # Initialize our trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=ds[\"train\"] if training_args.do_train else None,\n        eval_dataset=ds[\"validation\"] if training_args.do_eval else None,\n        processing_class=image_processor,\n        data_collator=collate_fn,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate()\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Write model card and (optionally) push to hub\n    kwargs = {\n        \"tasks\": \"masked-auto-encoding\",\n        \"dataset\": data_args.dataset_name,\n        \"tags\": [\"masked-auto-encoding\"],\n    }\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nThis code is part of the Hugging Face Transformers library, implemented in Python. It provides a comprehensive framework for fine-tuning pre-trained models such as ViTMAEForPreTraining on various image classification tasks.\n\nHere's a concise summary:\n\n1.  **Initialization and Model Setup**: The script starts by importing required libraries and initializing a new instance of the model (ViTMAEForPreTraining) or loading an existing one.\n\n2.  **Data Preparation and Preprocessing**: It then sets up data preparation strategies for training and evaluation, including transformations to apply on the images in the dataset.\n\n3.  **Trainer Implementation**: A Trainer object is instantiated with the model, training arguments, and other relevant details. The trainer is responsible for orchestrating training and saving model checkpoints, metrics, and state information.\n\n4.  **Training and Evaluation**: If required by the training configuration (do\\_train setting), it trains the model until completion or a specified number of epochs. It also evaluates the trained model on a validation dataset.\n\n5.  **Logging and Hub Operations**: After training or evaluation is complete, the script logs specific metrics like model performance, saves key metrics to a file, and optionally pushes the fine-tuned model to Hugging Face's Model Hub for sharing.\n\nThe code is written in Python, utilizes the Transformers library by Hugging Face Inc., and is designed with modular, expandable functionality allowing users to easily implement their preferred data preparation strategies and training protocols."}


created_at: 2025-02-05 09:13:35
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py:401-403
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 403, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mae.py", "name": "_mp_fn", "start_line": 401, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Function Summary: `_mp_fn`**\n\n* **Purpose:** Invokes the `main()` function using xla_spawn (TPU) multi-processing.\n\n* **Key Implementation:** Calls external library (`xla_spawn`) to utilize TPUs for parallel execution.\n\n* **Notable Function Signature:** `_mp_fn(index)`\n\t+ `index`: Input parameter passed to the function, not explicitly used within the code snippet.\n \n* **External Dependencies:**\n\t+ `main()`\n\t+ `xla_spawn` (TPUs)\n     No explicit dependencies on internal classes/functions."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py:196-229
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 229, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py", "name": "MaskGenerator", "start_line": 196, "type": "class", "chroma:document": "CODE:\n\nclass MaskGenerator:\n    \"\"\"\n    A class to generate boolean masks for the pretraining task.\n\n    A mask is a 1D tensor of shape (model_patch_size**2,) where the value is either 0 or 1,\n    where 1 indicates \"masked\".\n    \"\"\"\n\n    def __init__(self, input_size=192, mask_patch_size=32, model_patch_size=4, mask_ratio=0.6):\n        self.input_size = input_size\n        self.mask_patch_size = mask_patch_size\n        self.model_patch_size = model_patch_size\n        self.mask_ratio = mask_ratio\n\n        if self.input_size % self.mask_patch_size != 0:\n            raise ValueError(\"Input size must be divisible by mask patch size\")\n        if self.mask_patch_size % self.model_patch_size != 0:\n            raise ValueError(\"Mask patch size must be divisible by model patch size\")\n\n        self.rand_size = self.input_size // self.mask_patch_size\n        self.scale = self.mask_patch_size // self.model_patch_size\n\n        self.token_count = self.rand_size**2\n        self.mask_count = int(np.ceil(self.token_count * self.mask_ratio))\n\n    def __call__(self):\n        mask_idx = np.random.permutation(self.token_count)[: self.mask_count]\n        mask = np.zeros(self.token_count, dtype=int)\n        mask[mask_idx] = 1\n\n        mask = mask.reshape((self.rand_size, self.rand_size))\n        mask = mask.repeat(self.scale, axis=0).repeat(self.scale, axis=1)\n\n\n\nSUMMARY:\n**Class Summary: `MaskGenerator`**\n\n**Main Purpose and Functionality**\nThe `MaskGenerator` class generates 1D boolean masks for pretraining tasks. It creates a mask with shape `(model_patch_size**2,)` where values are either 0 or 1, indicating \"masked\" or not masked.\n\n**Key Implementation Details**\n\n* `__init__`: Initializes the generator with input size, mask patch size, model patch size, and mask ratio.\n\t+ Checks divisibility of input size by mask patch size and mask patch size by model patch size.\n\t+ Calculates token count and scaled mask count based on input size, mask patch size, and model patch size.\n* `__call__`: Generates a random mask with the specified shape.\n\t+ Creates a permutation of indices for masking tokens.\n\t+ Zero-pads and repeats the permutation to create the full mask.\n\n**Notable Dependencies and Requirements**\n\n* Imports:\n\t+ `torch` (for tensor creation)\n\t+ `np` (for NumPy arrays and functions)\n* Interfaces/Classes inheriting from this class: None\n* Inherited interfaces: None"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py:232-235
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 235, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py", "name": "collate_fn", "start_line": 232, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(examples):\n    pixel_values = torch.stack([example[\"pixel_values\"] for example in examples])\n    mask = torch.stack([example[\"mask\"] for example in examples])\n\n\nSUMMARY:\n**Function Summary**\n\nThe `collate_fn` function collates input data for training, combining all pixel values and mask tensors from multiple examples into a single batch.\n\n**Key Details:**\n\n* Combines `pixel_values` and `mask` tensor fields from each example using `torch.stack()`.\n* Returns a dictionary with the collated `pixel_values` as 'pixel_values' key and booleans indicating masked positions as 'bool_masked_pos' key.\n\n**Notable Dependencies/Requirements:**\n\n* `torch`: Required for tensor operations (e.g., `torch.stack()`)."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py:238-478
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 478, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/image-pretraining/run_mim.py", "name": "main", "start_line": 238, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_mim\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Initialize our dataset.\n    ds = load_dataset(\n        data_args.dataset_name,\n        data_args.dataset_config_name,\n        data_files=data_args.data_files,\n        cache_dir=model_args.cache_dir,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # If we don't have a validation split, split off a percentage of train as validation.\n    data_args.train_val_split = None if \"validation\" in ds.keys() else data_args.train_val_split\n    if isinstance(data_args.train_val_split, float) and data_args.train_val_split > 0.0:\n        split = ds[\"train\"].train_test_split(data_args.train_val_split)\n        ds[\"train\"] = split[\"train\"]\n        ds[\"validation\"] = split[\"test\"]\n\n    # Create config\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config_kwargs = {\n        \"cache_dir\": model_args.cache_dir,\n        \"revision\": model_args.model_revision,\n        \"token\": model_args.token,\n        \"trust_remote_code\": model_args.trust_remote_code,\n    }\n    if model_args.config_name_or_path:\n        config = AutoConfig.from_pretrained(model_args.config_name_or_path, **config_kwargs)\n    elif model_args.model_name_or_path:\n        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n    else:\n        config = CONFIG_MAPPING[model_args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n        if model_args.config_overrides is not None:\n            logger.info(f\"Overriding config: {model_args.config_overrides}\")\n            config.update_from_string(model_args.config_overrides)\n            logger.info(f\"New config: {config}\")\n\n    # make sure the decoder_type is \"simmim\" (only relevant for BEiT)\n    if hasattr(config, \"decoder_type\"):\n        config.decoder_type = \"simmim\"\n\n    # adapt config\n    model_args.image_size = model_args.image_size if model_args.image_size is not None else config.image_size\n    model_args.patch_size = model_args.patch_size if model_args.patch_size is not None else config.patch_size\n    model_args.encoder_stride = (\n        model_args.encoder_stride if model_args.encoder_stride is not None else config.encoder_stride\n    )\n\n    config.update(\n        {\n            \"image_size\": model_args.image_size,\n            \"patch_size\": model_args.patch_size,\n            \"encoder_stride\": model_args.encoder_stride,\n        }\n    )\n\n    # create image processor\n    if model_args.image_processor_name:\n        image_processor = AutoImageProcessor.from_pretrained(model_args.image_processor_name, **config_kwargs)\n    elif model_args.model_name_or_path:\n        image_processor = AutoImageProcessor.from_pretrained(model_args.model_name_or_path, **config_kwargs)\n    else:\n        IMAGE_PROCESSOR_TYPES = {\n            conf.model_type: image_processor_class for conf, image_processor_class in IMAGE_PROCESSOR_MAPPING.items()\n        }\n        image_processor = IMAGE_PROCESSOR_TYPES[model_args.model_type]()\n\n    # create model\n    if model_args.model_name_or_path:\n        model = AutoModelForMaskedImageModeling.from_pretrained(\n            model_args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n            config=config,\n            cache_dir=model_args.cache_dir,\n            revision=model_args.model_revision,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForMaskedImageModeling.from_config(config, trust_remote_code=model_args.trust_remote_code)\n\n    if training_args.do_train:\n        column_names = ds[\"train\"].column_names\n    else:\n        column_names = ds[\"validation\"].column_names\n\n    if data_args.image_column_name is not None:\n        image_column_name = data_args.image_column_name\n    elif \"image\" in column_names:\n        image_column_name = \"image\"\n    elif \"img\" in column_names:\n        image_column_name = \"img\"\n    else:\n        image_column_name = column_names[0]\n\n    # transformations as done in original SimMIM paper\n    # source: https://github.com/microsoft/SimMIM/blob/main/data/data_simmim.py\n    transforms = Compose(\n        [\n            Lambda(lambda img: img.convert(\"RGB\") if img.mode != \"RGB\" else img),\n            RandomResizedCrop(model_args.image_size, scale=(0.67, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)),\n            RandomHorizontalFlip(),\n            ToTensor(),\n            Normalize(mean=image_processor.image_mean, std=image_processor.image_std),\n        ]\n    )\n\n    # create mask generator\n    mask_generator = MaskGenerator(\n        input_size=model_args.image_size,\n        mask_patch_size=data_args.mask_patch_size,\n        model_patch_size=model_args.patch_size,\n        mask_ratio=data_args.mask_ratio,\n    )\n\n    def preprocess_images(examples):\n        \"\"\"Preprocess a batch of images by applying transforms + creating a corresponding mask, indicating\n        which patches to mask.\"\"\"\n\n        examples[\"pixel_values\"] = [transforms(image) for image in examples[image_column_name]]\n        examples[\"mask\"] = [mask_generator() for i in range(len(examples[image_column_name]))]\n\n        return examples\n\n    if training_args.do_train:\n        if \"train\" not in ds:\n            raise ValueError(\"--do_train requires a train dataset\")\n        if data_args.max_train_samples is not None:\n            ds[\"train\"] = ds[\"train\"].shuffle(seed=training_args.seed).select(range(data_args.max_train_samples))\n        # Set the training transforms\n        ds[\"train\"].set_transform(preprocess_images)\n\n    if training_args.do_eval:\n        if \"validation\" not in ds:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        if data_args.max_eval_samples is not None:\n            ds[\"validation\"] = (\n                ds[\"validation\"].shuffle(seed=training_args.seed).select(range(data_args.max_eval_samples))\n            )\n        # Set the validation transforms\n        ds[\"validation\"].set_transform(preprocess_images)\n\n    # Initialize our trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=ds[\"train\"] if training_args.do_train else None,\n        eval_dataset=ds[\"validation\"] if training_args.do_eval else None,\n        processing_class=image_processor,\n        data_collator=collate_fn,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate()\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Write model card and (optionally) push to hub\n    kwargs = {\n        \"finetuned_from\": model_args.model_name_or_path,\n        \"tasks\": \"masked-image-modeling\",\n        \"dataset\": data_args.dataset_name,\n        \"tags\": [\"masked-image-modeling\"],\n    }\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nHere is a concise summary of the provided code:\n\nThis is the implementation of the SimMIM model, which is a type of masked image modeling task. The script defines a dataset class for loading and preprocessing images, a data collator function to transform pairs of image patches into input tensors, and a `Trainer` class to perform training and evaluation.\n\nThe main pipeline of this code includes:\n\n1. Loading the dataset and preprocessing images using transforms.\n2. Creating a mask generator to create masks for masked image models.\n3. Defining a data collator function to transform batches of image patches into input tensors for the model.\n4. Training the model using a `Trainer` class, which handles model training, evaluation, saving, and loading checkpoints.\n\nThe script also includes support for fine-tuning pre-trained models and pushing the trained models to Hugging Face's model hub.\n\nOverall, this code provides a comprehensive implementation of masked image modeling, including data preprocessing, transformation, and training."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:59-217

metadata:
{"docstring": "", "end_line": 217, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "parse_args", "start_line": 59, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model for instance segmentation task\")\n\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to a pretrained model or model identifier from huggingface.co/models.\",\n        default=\"facebook/mask2former-swin-tiny-coco-instance\",\n    )\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        help=\"Name of the dataset on the hub.\",\n        default=\"qubvel-hf/ade20k-mini\",\n    )\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--image_height\",\n        type=int,\n        default=384,\n        help=\"The height of the images to feed the model.\",\n    )\n    parser.add_argument(\n        \"--image_width\",\n        type=int,\n        default=384,\n        help=\"The width of the images to feed the model.\",\n    )\n    parser.add_argument(\n        \"--do_reduce_labels\",\n        action=\"store_true\",\n        help=\"Whether to reduce the number of labels by removing the background class.\",\n    )\n    parser.add_argument(\n        \"--cache_dir\",\n        type=str,\n        help=\"Path to a folder in which the model and dataset will be cached.\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--dataloader_num_workers\",\n        type=int,\n        default=4,\n        help=\"Number of workers to use for the dataloaders.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\n        \"--adam_beta1\",\n        type=float,\n        default=0.9,\n        help=\"Beta1 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_beta2\",\n        type=float,\n        default=0.999,\n        help=\"Beta2 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_epsilon\",\n        type=float,\n        default=1e-8,\n        help=\"Epsilon for AdamW optimizer\",\n    )\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        required=False,\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.push_to_hub or args.with_tracking:\n        if args.output_dir is None:\n            raise ValueError(\n                \"Need an `output_dir` to create a repo when `--push_to_hub` or `with_tracking` is specified.\"\n            )\n\n    if args.output_dir is not None:\n        os.makedirs(args.output_dir, exist_ok=True)\n\n\n\nSUMMARY:\n**Function Summary**\n\nThe provided function `parse_args()` defines and parses command-line arguments for fine-tuning a pre-trained transformer model for instance segmentation using Hugging Face's transformers library.\n\n### Main Purpose and Functionality\n\nThe primary purpose of this function is to parse and return the command-line arguments configured by the user. It uses the `argparse` library to create an argument parser with various input parameters, such as model name or path, dataset name, image dimensions, batch sizes, learning rate, and schedule type.\n\n### Key Implementation Details\n\n* Notable algorithms and designs:\n\t+ The function uses a combination of predefined arguments from the transformers library, with some customizable options.\n\t+ The `SchedulerType` parameter is mapped to different learning rate schedulers.\n* Important method signatures and interfaces:\n\t+ None; the function relies on the argparse library for its structure.\n\n### Notable Dependencies or Requirements\n\n* The following external dependencies are required:\n\t+ `huggingface` library for accessing pre-trained models and transformers\n\t+ `argparse` library for parsing command-line arguments\n\t+ `os` library for creating directories\n\n### Summary Statistics\n\nThe function has 26 unique argument parameters, with default values provided for most of them. The parser performs basic sanity checks on the input configuration before returning the parsed arguments.\n\n```markdown\n# Import statements (none)\n\n# Function definition\ndef parse_args():\n    ...\n    \n    # Parsed command-line arguments\n    args = parser.parse_args()\n    ...\n\n# Parameters summary (26 unique parameters)\n- Model name/path:\n  --model_name_or_path\n- Dataset name:\n  --dataset_name\n- Image dimensions:\n  --image_height, --image_width\n- Batch sizes:\n  --per_device_train_batch_size, --per_device_eval_batch_size\n- Learning rate:\n  --learning_rate\n- Schedule type:\n  --lr_scheduler_type\n\n# External dependencies (huggingface, argparse, os)\n```"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:220-258

metadata:
{"docstring": "", "end_line": 258, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "augment_and_transform_batch", "start_line": 220, "type": "function", "chroma:document": "CODE:\n\ndef augment_and_transform_batch(\n    examples: Mapping[str, Any], transform: A.Compose, image_processor: AutoImageProcessor\n) -> BatchFeature:\n    batch = {\n        \"pixel_values\": [],\n        \"mask_labels\": [],\n        \"class_labels\": [],\n    }\n\n    for pil_image, pil_annotation in zip(examples[\"image\"], examples[\"annotation\"]):\n        image = np.array(pil_image)\n        semantic_and_instance_masks = np.array(pil_annotation)[..., :2]\n\n        # Apply augmentations\n        output = transform(image=image, mask=semantic_and_instance_masks)\n\n        aug_image = output[\"image\"]\n        aug_semantic_and_instance_masks = output[\"mask\"]\n        aug_instance_mask = aug_semantic_and_instance_masks[..., 1]\n\n        # Create mapping from instance id to semantic id\n        unique_semantic_id_instance_id_pairs = np.unique(aug_semantic_and_instance_masks.reshape(-1, 2), axis=0)\n        instance_id_to_semantic_id = {\n            instance_id: semantic_id for semantic_id, instance_id in unique_semantic_id_instance_id_pairs\n        }\n\n        # Apply the image processor transformations: resizing, rescaling, normalization\n        model_inputs = image_processor(\n            images=[aug_image],\n            segmentation_maps=[aug_instance_mask],\n            instance_id_to_semantic_id=instance_id_to_semantic_id,\n            return_tensors=\"pt\",\n        )\n\n        batch[\"pixel_values\"].append(model_inputs.pixel_values[0])\n        batch[\"mask_labels\"].append(model_inputs.mask_labels[0])\n        batch[\"class_labels\"].append(model_inputs.class_labels[0])\n\n\n\nSUMMARY:\n**Function Summary**\n\n`augment_and_transform_batch`: Transform medical images with associated masks and annotations, applying augmentations and processing transforms.\n\n**Key Functions/Classes:**\n\n* `A.Compose()`: Compose data augmentation transformations.\n* `AutoImageProcessor`.\n* `np.array()`: Convert PIL image to NumPy array.\n* `transform()`: Apply data augmentation transformation to input data.\n\n**Important Method Signatures:**\n\n* `image_processor(images, segmentation_maps, instance_id_to_semantic_id, return_tensors)`\n\n**Key Dependencies:**\n\n* `pandas` (implicit through `Mapping[str, Any]`)\n* `torch`\n* ` numpy`"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:261-268

metadata:
{"docstring": "", "end_line": 268, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "collate_fn", "start_line": 261, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(examples):\n    batch = {}\n    batch[\"pixel_values\"] = torch.stack([example[\"pixel_values\"] for example in examples])\n    batch[\"class_labels\"] = [example[\"class_labels\"] for example in examples]\n    batch[\"mask_labels\"] = [example[\"mask_labels\"] for example in examples]\n    if \"pixel_mask\" in examples[0]:\n        batch[\"pixel_mask\"] = torch.stack([example[\"pixel_mask\"] for example in examples])\n\n\nSUMMARY:\n**Function Summary:**\n\n`collate_fn` is a function that collates and batches input data (`examples`) into a single dictionary. It takes three key fields from each example:\n\n- `pixel_values`\n- `class_labels`\n- `mask_labels`\n\nOptional depending on the presence of \"pixel_mask\" in the examples.\n\n**Key Implementation Details:**\n\n- Batches pixel values, class labels, and mask labels using PyTorch's `stack` function.\n- Returns a dictionary containing these batches with keys 'pixel_values', 'class_labels', 'mask_labels' and optionally 'pixel_mask'.\n\nNotable Interfaces/Dependencies:\n\n- PyTorch (`torch`)."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:271-279

metadata:
{"docstring": "", "end_line": 279, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "nested_cpu", "start_line": 271, "type": "function", "chroma:document": "CODE:\n\ndef nested_cpu(tensors):\n    if isinstance(tensors, (list, tuple)):\n        return type(tensors)(nested_cpu(t) for t in tensors)\n    elif isinstance(tensors, Mapping):\n        return type(tensors)({k: nested_cpu(t) for k, t in tensors.items()})\n    elif isinstance(tensors, torch.Tensor):\n        return tensors.cpu().detach()\n    else:\n\n\nSUMMARY:\n**Function Summary: `nested_cpu`**\n\nThe `nested_cpu` function recursively traverses the data structure passed to it, applying CPU operations on each element. It handles three types of inputs:\n\n1. **List/Tuple**: Recursively calls itself on each element and returns a new list/tuple with the results.\n2. **Dictionary (Mapping)**: Recursively calls itself on each value and returns a new dictionary with the original keys mapped to the results.\n3. **PyTorch Tensor**: Detach the tensor from the GPU (if any) and return it as a CPU tensor.\n\n**Key Implementation Details:**\n\n* Uses `isinstance` to check the type of input.\n* Returns the original input if it's not an instance of the expected types.\n* Uses list expression and dictionary comprehension for concise iteration over elements.\n\n**Notable Dependencies/Requirements:**\n\n* PyTorch (`torch`) is a dependency."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:282-354

metadata:
{"docstring": "", "end_line": 354, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "evaluation_loop", "start_line": 282, "type": "function", "chroma:document": "CODE:\n\ndef evaluation_loop(model, image_processor, accelerator: Accelerator, dataloader, id2label):\n    metric = MeanAveragePrecision(iou_type=\"segm\", class_metrics=True)\n\n    for inputs in tqdm(dataloader, total=len(dataloader), disable=not accelerator.is_local_main_process):\n        with torch.no_grad():\n            outputs = model(**inputs)\n\n        inputs = accelerator.gather_for_metrics(inputs)\n        inputs = nested_cpu(inputs)\n\n        outputs = accelerator.gather_for_metrics(outputs)\n        outputs = nested_cpu(outputs)\n\n        # For metric computation we need to provide:\n        #  - targets in a form of list of dictionaries with keys \"masks\", \"labels\"\n        #  - predictions in a form of list of dictionaries with keys \"masks\", \"labels\", \"scores\"\n\n        post_processed_targets = []\n        post_processed_predictions = []\n        target_sizes = []\n\n        # Collect targets\n        for masks, labels in zip(inputs[\"mask_labels\"], inputs[\"class_labels\"]):\n            post_processed_targets.append(\n                {\n                    \"masks\": masks.to(dtype=torch.bool),\n                    \"labels\": labels,\n                }\n            )\n            target_sizes.append(masks.shape[-2:])\n\n        # Collect predictions\n        post_processed_output = image_processor.post_process_instance_segmentation(\n            outputs,\n            threshold=0.0,\n            target_sizes=target_sizes,\n            return_binary_maps=True,\n        )\n\n        for image_predictions, target_size in zip(post_processed_output, target_sizes):\n            if image_predictions[\"segments_info\"]:\n                post_processed_image_prediction = {\n                    \"masks\": image_predictions[\"segmentation\"].to(dtype=torch.bool),\n                    \"labels\": torch.tensor([x[\"label_id\"] for x in image_predictions[\"segments_info\"]]),\n                    \"scores\": torch.tensor([x[\"score\"] for x in image_predictions[\"segments_info\"]]),\n                }\n            else:\n                # for void predictions, we need to provide empty tensors\n                post_processed_image_prediction = {\n                    \"masks\": torch.zeros([0, *target_size], dtype=torch.bool),\n                    \"labels\": torch.tensor([]),\n                    \"scores\": torch.tensor([]),\n                }\n            post_processed_predictions.append(post_processed_image_prediction)\n\n        # Update metric for batch targets and predictions\n        metric.update(post_processed_predictions, post_processed_targets)\n\n    # Compute metrics\n    metrics = metric.compute()\n\n    # Replace list of per class metrics with separate metric for each class\n    classes = metrics.pop(\"classes\")\n    map_per_class = metrics.pop(\"map_per_class\")\n    mar_100_per_class = metrics.pop(\"mar_100_per_class\")\n    for class_id, class_map, class_mar in zip(classes, map_per_class, mar_100_per_class):\n        class_name = id2label[class_id.item()] if id2label is not None else class_id.item()\n        metrics[f\"map_{class_name}\"] = class_map\n        metrics[f\"mar_100_{class_name}\"] = class_mar\n\n    metrics = {k: round(v.item(), 4) for k, v in metrics.items()}\n\n\n\nSUMMARY:\n**Evaluation Loop Function Summary**\n\n### Main Purpose\nThe `evaluation_loop` function evaluates the performance of a model using object detection metrics. It takes in various inputs, including a model, image processor, accelerator, dataloader, and id2label dictionary.\n\n```python\ndef evaluation_loop(model: Model, image_processor: ImageProcessor, accelerator: Accelerator,\n                    dataloader: DataLoader, id2label: dict):\n    ...\n```\n\n### Key Implementation Details\n\n* The function uses the `torch.no_grad()` context to disable gradient computation.\n* It applies model inference using the `model(**inputs)` syntax, where inputs are processed via the `accelerator`.\n* The `image_processor.post_process_instance_segmentation()` function is used to post-process output masks and labels.\n\nKey method signatures:\n\n* `model(**inputs)`\n* `image_processor.post_process_instance_segmentation(outputs, ...`\n\n### Notable Dependencies or Requirements\n\n* The following modules are used:\n\t+ `torch`\n\t+ `tqdm` (for progress bar)\n\t+ `matplotlib` (not explicitly mentioned, but assumed for plotting)\n\nInterfaces and classes:\n\n* `Accelerator`: provides methods for gathering data and tracking metrics\n* `Model`: provides the inference method\n\n### External Dependencies\n\n* The following external dependencies are used:\n\t+ `dataloader` module (for loading data)\n\t+ `id2label` dictionary (mapping class IDs to class names)\n\nNote: The summary preserves exact class and function names, provides a concise description of the main purpose, key implementation details, and notable dependencies or requirements."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:357-372

metadata:
{"docstring": "", "end_line": 372, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "setup_logging", "start_line": 357, "type": "function", "chroma:document": "CODE:\n\ndef setup_logging(accelerator: Accelerator) -> None:\n    \"\"\"Setup logging according to `training_args`.\"\"\"\n\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n        logger.setLevel(logging.INFO)\n    else:\n        datasets.utils.logging.set_verbosity_error()\n\n\nSUMMARY:\n**Summary:**\n\n### Functionality:\n\nThe `setup_logging` function configures the logging system for a deep learning model, setting up a basic logging format and handler. It also adapts to whether the code is running on a local machine or remote server, adjusting verbosity levels accordingly.\n\n### Main Purpose:\n\nPrimary purpose: To establish a standardized logging configuration.\n \n### Key Implementation Details:\n- Sets logging format and date format.\n- Configures stream handler for output on stdout.\n- Adjusts verbosity based on local/main process determination.\n\n### Notable Dependencies:\n`logging`, `sys`, `datasets.utils.logging`, `transformers.utils.logging`\n\nNote: accelerator interface is referenced by `Accelerator`."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:375-398

metadata:
{"docstring": "", "end_line": 398, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "handle_repository_creation", "start_line": 375, "type": "function", "chroma:document": "CODE:\n\ndef handle_repository_creation(accelerator: Accelerator, args: argparse.Namespace):\n    \"\"\"Create a repository for the model and dataset if `args.push_to_hub` is set.\"\"\"\n\n    repo_id = None\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n\n\nSUMMARY:\n**Summary**\n\n`handle_repository_creation` function:\n---------------------------\n\n### Main Purpose and Functionality\n\nCreates a repository for the model and dataset based on user input. If `push_to_hub` is set, retrieves an existing repository or creates a new one; otherwise, creates a local directory.\n\n### Key Implementation Details\n\n* Retrieves repository name from `args.hub_model_id` or constructs it from `output_dir`.\n* Creates a new repository using Hugging Face API (`HfApi`) if `push_to_hub` is set.\n* If no repository exists, generates a `.gitignore` file with specific pattern matches.\n\n### Notable Dependencies and Requirements\n\n* `argparse.Namespace`: Argument parsing\n* `HuggingFaceAPI`: Repository creation service ( requires `hub_token`)\n* `Path`, `os.path.join`, `open`: File system utilities for handling directories and files."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py:401-739

metadata:
{"docstring": "", "end_line": 739, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation_no_trainer.py", "name": "main", "start_line": 401, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_instance_segmentation_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n    setup_logging(accelerator)\n\n    # If passed along, set the training seed now.\n    # We set device_specific to True as we want different data augmentation per device.\n    if args.seed is not None:\n        set_seed(args.seed, device_specific=True)\n\n    # Create repository if push ot hub is specified\n    repo_id = handle_repository_creation(accelerator, args)\n\n    if args.push_to_hub:\n        api = HfApi()\n\n    # ------------------------------------------------------------------------------------------------\n    # Load dataset, prepare splits\n    # ------------------------------------------------------------------------------------------------\n\n    # In distributed training, the load_dataset function guarantees that only one local process can concurrently\n    # download the dataset.\n    dataset = load_dataset(args.dataset_name, cache_dir=args.cache_dir, trust_remote_code=args.trust_remote_code)\n\n    # We need to specify the label2id mapping for the model\n    # it is a mapping from semantic class name to class index.\n    # In case your dataset does not provide it, you can create it manually:\n    # label2id = {\"background\": 0, \"cat\": 1, \"dog\": 2}\n    label2id = dataset[\"train\"][0][\"semantic_class_to_id\"]\n\n    if args.do_reduce_labels:\n        label2id = {name: idx for name, idx in label2id.items() if idx != 0}  # remove background class\n        label2id = {name: idx - 1 for name, idx in label2id.items()}  # shift class indices by -1\n\n    id2label = {v: k for k, v in label2id.items()}\n\n    # ------------------------------------------------------------------------------------------------\n    # Load pretrained model and image processor\n    # ------------------------------------------------------------------------------------------------\n    model = AutoModelForUniversalSegmentation.from_pretrained(\n        args.model_name_or_path,\n        label2id=label2id,\n        id2label=id2label,\n        ignore_mismatched_sizes=True,\n        token=args.hub_token,\n    )\n\n    image_processor = AutoImageProcessor.from_pretrained(\n        args.model_name_or_path,\n        do_resize=True,\n        size={\"height\": args.image_height, \"width\": args.image_width},\n        do_reduce_labels=args.do_reduce_labels,\n        reduce_labels=args.do_reduce_labels,  # TODO: remove when mask2former support `do_reduce_labels`\n        token=args.hub_token,\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define image augmentations and dataset transforms\n    # ------------------------------------------------------------------------------------------------\n    train_augment_and_transform = A.Compose(\n        [\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.5),\n            A.HueSaturationValue(p=0.1),\n        ],\n    )\n    validation_transform = A.Compose(\n        [A.NoOp()],\n    )\n\n    # Make transform functions for batch and apply for dataset splits\n    train_transform_batch = partial(\n        augment_and_transform_batch, transform=train_augment_and_transform, image_processor=image_processor\n    )\n    validation_transform_batch = partial(\n        augment_and_transform_batch, transform=validation_transform, image_processor=image_processor\n    )\n\n    with accelerator.main_process_first():\n        dataset[\"train\"] = dataset[\"train\"].with_transform(train_transform_batch)\n        dataset[\"validation\"] = dataset[\"validation\"].with_transform(validation_transform_batch)\n\n    dataloader_common_args = {\n        \"num_workers\": args.dataloader_num_workers,\n        \"persistent_workers\": True,\n        \"collate_fn\": collate_fn,\n    }\n    train_dataloader = DataLoader(\n        dataset[\"train\"], shuffle=True, batch_size=args.per_device_train_batch_size, **dataloader_common_args\n    )\n    valid_dataloader = DataLoader(\n        dataset[\"validation\"], shuffle=False, batch_size=args.per_device_eval_batch_size, **dataloader_common_args\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define optimizer, scheduler and prepare everything with the accelerator\n    # ------------------------------------------------------------------------------------------------\n\n    # Optimizer\n    optimizer = torch.optim.AdamW(\n        list(model.parameters()),\n        lr=args.learning_rate,\n        betas=[args.adam_beta1, args.adam_beta2],\n        eps=args.adam_epsilon,\n    )\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, valid_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, valid_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"instance_segmentation_no_trainer\", experiment_config)\n\n    # ------------------------------------------------------------------------------------------------\n    # Run training with evaluation on each epoch\n    # ------------------------------------------------------------------------------------------------\n\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(dataset['train'])}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n                    if args.push_to_hub and epoch < args.num_train_epochs - 1:\n                        accelerator.wait_for_everyone()\n                        unwrapped_model = accelerator.unwrap_model(model)\n                        unwrapped_model.save_pretrained(\n                            args.output_dir,\n                            is_main_process=accelerator.is_main_process,\n                            save_function=accelerator.save,\n                        )\n                        if accelerator.is_main_process:\n                            image_processor.save_pretrained(args.output_dir)\n                            api.upload_folder(\n                                repo_id=repo_id,\n                                commit_message=f\"Training in progress epoch {epoch}\",\n                                folder_path=args.output_dir,\n                                repo_type=\"model\",\n                                token=args.hub_token,\n                            )\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        logger.info(\"***** Running evaluation *****\")\n        metrics = evaluation_loop(model, image_processor, accelerator, valid_dataloader, id2label)\n\n        logger.info(f\"epoch {epoch}: {metrics}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    **metrics,\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                image_processor.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    # ------------------------------------------------------------------------------------------------\n    # Run evaluation on test dataset and save the model\n    # ------------------------------------------------------------------------------------------------\n\n    logger.info(\"***** Running evaluation on test dataset *****\")\n    metrics = evaluation_loop(model, image_processor, accelerator, valid_dataloader, id2label)\n    metrics = {f\"test_{k}\": v for k, v in metrics.items()}\n\n    logger.info(f\"Test metrics: {metrics}\")\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n                json.dump(metrics, f, indent=2)\n\n            image_processor.save_pretrained(args.output_dir)\n\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                    ignore_patterns=[\"epoch_*\"],\n\n\nSUMMARY:\n**Summary: Training, Evaluation, and Model Saving Script for Object Detection Models**\n\nThis script is used to train, evaluate, and save object detection models using the Hugging Face Transformers library. The script includes the following main components:\n\n1. **Model Training**: The script initializes the model, tokenizer, and optimizer, then trains the model on the training dataset.\n2. **Evaluation Loop**: During each epoch, the script evaluates the model on a validation dataset to track metrics such as accuracy, precision, and recall.\n3. **Saving and Versioning**: The script saves the trained model at regular intervals (optional) or after all epochs are completed.\n\nThe script includes various parameters that can be adjusted to optimize training and evaluation, such as:\n\n* `output_dir`: the directory where the saved models will be stored\n* `push_to_hub`: a boolean flag indicating whether to upload the saved model to Hugging Face Hub\n* `repo_id` and `hub_token`: tokens for authenticating with Hugging Face Hub\n* `checkpointing_steps`: the frequency at which to save the model (integer, \"epoch\", or \"step\")\n\nThe script outputs various logging statements during training and evaluation, including metrics on the validation dataset. Additionally, it creates a final output directory containing the saved model and any necessary metadata files.\n\n**Example Use Case:**\n\nTo run this script, navigate to the directory containing the script and modify the `output_dir`, `push_to_hub`, and other parameters as needed. Then, execute the script using Python:\n```python\npython train.py --config_file config.json\n```\nReplace `train.py` with the actual filename of your script, and `config.json` with a configuration file that specifies any desired parameters."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:102-140

metadata:
{"docstring": "", "end_line": 140, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "augment_and_transform_batch", "start_line": 102, "type": "function", "chroma:document": "CODE:\n\ndef augment_and_transform_batch(\n    examples: Mapping[str, Any], transform: A.Compose, image_processor: AutoImageProcessor\n) -> BatchFeature:\n    batch = {\n        \"pixel_values\": [],\n        \"mask_labels\": [],\n        \"class_labels\": [],\n    }\n\n    for pil_image, pil_annotation in zip(examples[\"image\"], examples[\"annotation\"]):\n        image = np.array(pil_image)\n        semantic_and_instance_masks = np.array(pil_annotation)[..., :2]\n\n        # Apply augmentations\n        output = transform(image=image, mask=semantic_and_instance_masks)\n\n        aug_image = output[\"image\"]\n        aug_semantic_and_instance_masks = output[\"mask\"]\n        aug_instance_mask = aug_semantic_and_instance_masks[..., 1]\n\n        # Create mapping from instance id to semantic id\n        unique_semantic_id_instance_id_pairs = np.unique(aug_semantic_and_instance_masks.reshape(-1, 2), axis=0)\n        instance_id_to_semantic_id = {\n            instance_id: semantic_id for semantic_id, instance_id in unique_semantic_id_instance_id_pairs\n        }\n\n        # Apply the image processor transformations: resizing, rescaling, normalization\n        model_inputs = image_processor(\n            images=[aug_image],\n            segmentation_maps=[aug_instance_mask],\n            instance_id_to_semantic_id=instance_id_to_semantic_id,\n            return_tensors=\"pt\",\n        )\n\n        batch[\"pixel_values\"].append(model_inputs.pixel_values[0])\n        batch[\"mask_labels\"].append(model_inputs.mask_labels[0])\n        batch[\"class_labels\"].append(model_inputs.class_labels[0])\n\n\n\nSUMMARY:\n**Function Summary: `augment_and_transform_batch`**\n\nMain Purpose and Functionality:\nThis function takes in a mapping of images and annotations, applies augmentations using `A.Compose`, processes the output with an image processor (`AutoImageProcessor`), and returns a batched feature object containing pixel values, mask labels, and class labels.\n\nKey Implementation Details:\n\n* Applies augmentations to input images using `transform`.\n* Creates a mapping from instance IDs to semantic IDs based on augmented masks.\n* Processes the augmented images with the image processor (`model_inputs`).\n* Appends processed features to a batched feature object (`batch`).\n\nNotable Dependencies and Requirements:\n\n* `A.Compose` for augmentation\n* `AutoImageProcessor` for image processing\n* `Mapping[str, Any]` for input examples\n* `np.array` and `numpy` for image manipulation\n\nReturn Type: `BatchFeature`\n\nNote: The summary preserves exact class and function names, and focuses on precise technical terms."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:143-150

metadata:
{"docstring": "", "end_line": 150, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "collate_fn", "start_line": 143, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(examples):\n    batch = {}\n    batch[\"pixel_values\"] = torch.stack([example[\"pixel_values\"] for example in examples])\n    batch[\"class_labels\"] = [example[\"class_labels\"] for example in examples]\n    batch[\"mask_labels\"] = [example[\"mask_labels\"] for example in examples]\n    if \"pixel_mask\" in examples[0]:\n        batch[\"pixel_mask\"] = torch.stack([example[\"pixel_mask\"] for example in examples])\n\n\nSUMMARY:\n**Collate Function Summary:**\n\nThe `collate_fn` function collates and preprocesses input data for model training. Its primary purpose is to group a list of dictionaries (`examples`) into batches, aggregating the corresponding values.\n\n**Key Implementation Details:**\n\n- Aggregates pixel values, class labels, and mask labels from individual examples using torch.stack.\n  \n**Notable Dependencies or Requirements:**\n\n- `torch`: used for stack operation on tensor data."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:159-167

metadata:
{"docstring": "", "end_line": 167, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "nested_cpu", "start_line": 159, "type": "function", "chroma:document": "CODE:\n\ndef nested_cpu(tensors):\n    if isinstance(tensors, (list, tuple)):\n        return type(tensors)(nested_cpu(t) for t in tensors)\n    elif isinstance(tensors, Mapping):\n        return type(tensors)({k: nested_cpu(t) for k, t in tensors.items()})\n    elif isinstance(tensors, torch.Tensor):\n        return tensors.cpu().detach()\n    else:\n\n\nSUMMARY:\n**Function Summary:**\n### `nested_cpu`\n\n#### Description\nRecursively traverses the input tensors data structure, transferring data to CPU and detaching it from GPU if present.\n\n#### Implementation Details\n\n*   Returns recursively transformed data structure (list/tuple or dict) with nested tensor elements converted to CPU.\n*   Applies transformation individually to torch.Tensor element using `cpu().detach()` method.\n*   Handles primitive tensors and unknown/unsupported types by returning the input unchanged.\n\n**Key Methods and Interfaces:**\n\n*   `type(tensors)(...)`: Creates a new data structure type wrapping recursively transformed elements.\n*   `tensors.cpu().detach()`: Converts torch.Tensor to CPU, detaching it from GPU."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:170-296

metadata:
{"docstring": "", "end_line": 296, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "Evaluator", "start_line": 170, "type": "class", "chroma:document": "CODE:\n\nclass Evaluator:\n    \"\"\"\n    Compute metrics for the instance segmentation task.\n    \"\"\"\n\n    def __init__(\n        self,\n        image_processor: AutoImageProcessor,\n        id2label: Mapping[int, str],\n        threshold: float = 0.0,\n    ):\n        \"\"\"\n        Initialize evaluator with image processor, id2label mapping and threshold for filtering predictions.\n\n        Args:\n            image_processor (AutoImageProcessor): Image processor for\n                `post_process_instance_segmentation` method.\n            id2label (Mapping[int, str]): Mapping from class id to class name.\n            threshold (float): Threshold to filter predicted boxes by confidence. Defaults to 0.0.\n        \"\"\"\n        self.image_processor = image_processor\n        self.id2label = id2label\n        self.threshold = threshold\n        self.metric = self.get_metric()\n\n    def get_metric(self):\n        metric = MeanAveragePrecision(iou_type=\"segm\", class_metrics=True)\n        return metric\n\n    def reset_metric(self):\n        self.metric.reset()\n\n    def postprocess_target_batch(self, target_batch) -> List[Dict[str, torch.Tensor]]:\n        \"\"\"Collect targets in a form of list of dictionaries with keys \"masks\", \"labels\".\"\"\"\n        batch_masks = target_batch[0]\n        batch_labels = target_batch[1]\n        post_processed_targets = []\n        for masks, labels in zip(batch_masks, batch_labels):\n            post_processed_targets.append(\n                {\n                    \"masks\": masks.to(dtype=torch.bool),\n                    \"labels\": labels,\n                }\n            )\n        return post_processed_targets\n\n    def get_target_sizes(self, post_processed_targets) -> List[List[int]]:\n        target_sizes = []\n        for target in post_processed_targets:\n            target_sizes.append(target[\"masks\"].shape[-2:])\n        return target_sizes\n\n    def postprocess_prediction_batch(self, prediction_batch, target_sizes) -> List[Dict[str, torch.Tensor]]:\n        \"\"\"Collect predictions in a form of list of dictionaries with keys \"masks\", \"labels\", \"scores\".\"\"\"\n\n        model_output = ModelOutput(class_queries_logits=prediction_batch[0], masks_queries_logits=prediction_batch[1])\n        post_processed_output = self.image_processor.post_process_instance_segmentation(\n            model_output,\n            threshold=self.threshold,\n            target_sizes=target_sizes,\n            return_binary_maps=True,\n        )\n\n        post_processed_predictions = []\n        for image_predictions, target_size in zip(post_processed_output, target_sizes):\n            if image_predictions[\"segments_info\"]:\n                post_processed_image_prediction = {\n                    \"masks\": image_predictions[\"segmentation\"].to(dtype=torch.bool),\n                    \"labels\": torch.tensor([x[\"label_id\"] for x in image_predictions[\"segments_info\"]]),\n                    \"scores\": torch.tensor([x[\"score\"] for x in image_predictions[\"segments_info\"]]),\n                }\n            else:\n                # for void predictions, we need to provide empty tensors\n                post_processed_image_prediction = {\n                    \"masks\": torch.zeros([0, *target_size], dtype=torch.bool),\n                    \"labels\": torch.tensor([]),\n                    \"scores\": torch.tensor([]),\n                }\n            post_processed_predictions.append(post_processed_image_prediction)\n\n        return post_processed_predictions\n\n    @torch.no_grad()\n    def __call__(self, evaluation_results: EvalPrediction, compute_result: bool = False) -> Mapping[str, float]:\n        \"\"\"\n        Update metrics with current evaluation results and return metrics if `compute_result` is True.\n\n        Args:\n            evaluation_results (EvalPrediction): Predictions and targets from evaluation.\n            compute_result (bool): Whether to compute and return metrics.\n\n        Returns:\n            Mapping[str, float]: Metrics in a form of dictionary {<metric_name>: <metric_value>}\n        \"\"\"\n        prediction_batch = nested_cpu(evaluation_results.predictions)\n        target_batch = nested_cpu(evaluation_results.label_ids)\n\n        # For metric computation we need to provide:\n        #  - targets in a form of list of dictionaries with keys \"masks\", \"labels\"\n        #  - predictions in a form of list of dictionaries with keys \"masks\", \"labels\", \"scores\"\n        post_processed_targets = self.postprocess_target_batch(target_batch)\n        target_sizes = self.get_target_sizes(post_processed_targets)\n        post_processed_predictions = self.postprocess_prediction_batch(prediction_batch, target_sizes)\n\n        # Compute metrics\n        self.metric.update(post_processed_predictions, post_processed_targets)\n\n        if not compute_result:\n            return\n\n        metrics = self.metric.compute()\n\n        # Replace list of per class metrics with separate metric for each class\n        classes = metrics.pop(\"classes\")\n        map_per_class = metrics.pop(\"map_per_class\")\n        mar_100_per_class = metrics.pop(\"mar_100_per_class\")\n        for class_id, class_map, class_mar in zip(classes, map_per_class, mar_100_per_class):\n            class_name = self.id2label[class_id.item()] if self.id2label is not None else class_id.item()\n            metrics[f\"map_{class_name}\"] = class_map\n            metrics[f\"mar_100_{class_name}\"] = class_mar\n\n        metrics = {k: round(v.item(), 4) for k, v in metrics.items()}\n\n        # Reset metric for next evaluation\n        self.reset_metric()\n\n\n\nSUMMARY:\n**Evaluator Class Summary**\n\nThe `Evaluator` class computes metrics for instance segmentation tasks. Its primary purpose is to process prediction and target batches, update metrics, and return computed values.\n\n**Key Implementation Details:**\n\n1.  **Post-processing**: The `postprocess_target_batch` method collects targets in a standardized format, while the `postprocess_prediction_batch` method post-processes predictions based on a given threshold.\n2.  **Metric Computation**: The `get_metric` method initializes the metric object, and the `__call__` method updates metrics with evaluation results using the `(metric.update()` method).\n3.  **Metric Aggregation**: Metrics are aggregated and transformed into a comprehensive dictionary containing class-specific values.\n\n**Notable Dependencies and Requirements:**\n\n1.  `ModelOutput`\n2.  `ImageProcessor` (from `AutoImageProcessor`)\n3.  `MeanAveragePrecision`\n4.  `torch` library\n\n    The dependency information indicates that the Evaluator uses specific models, image processors, and metrics to compute instance segmentation tasks.\n\n**Summary Metrics:**\n\n*   Instantiation: Class instantiation with necessary dependencies.\n*   Processing: Data transformation in both prediction and target batches.\n*   Evaluation: Computation of instance segmentation metrics.\n*   Results: Dictionary containing computed metric values."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:299-316

metadata:
{"docstring": "", "end_line": 316, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "setup_logging", "start_line": 299, "type": "function", "chroma:document": "CODE:\n\ndef setup_logging(training_args: TrainingArguments) -> None:\n    \"\"\"Setup logging according to `training_args`.\"\"\"\n\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n\n\nSUMMARY:\n**Function Summary: `setup_logging`**\n\n**Purpose and Functionality:**\nSets up logging configuration according to the provided `TrainingArguments`.\n\n**Implementation Details:**\n\n* Configures logging handlers with basic format string and date/time format\n* Sets verbosity level based on specified training arguments\n\t+ Enables default log level (info) if `should_log` is enabled\n\t+ Sets process-specific log level from training arguments\n* Enables explicit format and all default log handlers\n\n**Dependencies and Requirements:**\n\n* Imports `sys`, `logging`, and `transformers.utils.logging`\n* Uses external dependencies:\n\t+ `transformers`: specific logging utilities (set_verbosity, enable_default_handler, etc.)\n\t+ `logging`: basic logger functionality\n\t+ `sys`: access to system stdout"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:319-338

metadata:
{"docstring": "", "end_line": 338, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "find_last_checkpoint", "start_line": 319, "type": "function", "chroma:document": "CODE:\n\ndef find_last_checkpoint(training_args: TrainingArguments) -> Optional[str]:\n    \"\"\"Find the last checkpoint in the output directory according to parameters specified in `training_args`.\"\"\"\n\n    checkpoint = None\n    if training_args.resume_from_checkpoint is not None:\n        checkpoint = training_args.resume_from_checkpoint\n    elif os.path.isdir(training_args.output_dir) and not training_args.overwrite_output_dir:\n        checkpoint = get_last_checkpoint(training_args.output_dir)\n        if checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n\n\nSUMMARY:\n**Summary**\n\nThis is the `find_last_checkpoint` function, which finds and returns the last completed training check-point in the specified output directory based on provided parameters.\n\n**Key Implementation Details**\n\n- Checks for existing check-point by path (`training_args.resume_from_checkpoint`) or using `get_last_checkpoint`\n- Verifies output directory existence and emptiness before raising an error\n- Logs detection of a check-point when it exists but `--resume_from_checkpoint` is not specified\n\n**Dependencies and Requirements**\n\n- `TrainingArguments` object from the Hugging Face Transformers library\n- External dependency on `os.path.isdir`, `os.listdir`, and `logger`\n- Internal function call to `get_last_checkpoint`"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py:341-475

metadata:
{"docstring": "", "end_line": 475, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/instance-segmentation/run_instance_segmentation.py", "name": "main", "start_line": 341, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments\n    # or by passing the --help flag to this script.\n\n    parser = HfArgumentParser([Arguments, TrainingArguments])\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        args, training_args = parser.parse_args_into_dataclasses()\n\n    # Set default training arguments for instance segmentation\n    training_args.eval_do_concat_batches = False\n    training_args.batch_eval_metrics = True\n    training_args.remove_unused_columns = False\n\n    # # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_instance_segmentation\", args)\n\n    # Setup logging and log on each process the small summary:\n    setup_logging(training_args)\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Load last checkpoint from output_dir if it exists (and we are not overwriting it)\n    checkpoint = find_last_checkpoint(training_args)\n\n    # ------------------------------------------------------------------------------------------------\n    # Load dataset, prepare splits\n    # ------------------------------------------------------------------------------------------------\n\n    dataset = load_dataset(args.dataset_name, trust_remote_code=args.trust_remote_code)\n\n    # We need to specify the label2id mapping for the model\n    # it is a mapping from semantic class name to class index.\n    # In case your dataset does not provide it, you can create it manually:\n    # label2id = {\"background\": 0, \"cat\": 1, \"dog\": 2}\n    label2id = dataset[\"train\"][0][\"semantic_class_to_id\"]\n\n    if args.do_reduce_labels:\n        label2id = {name: idx for name, idx in label2id.items() if idx != 0}  # remove background class\n        label2id = {name: idx - 1 for name, idx in label2id.items()}  # shift class indices by -1\n\n    id2label = {v: k for k, v in label2id.items()}\n\n    # ------------------------------------------------------------------------------------------------\n    # Load pretrained config, model and image processor\n    # ------------------------------------------------------------------------------------------------\n    model = AutoModelForUniversalSegmentation.from_pretrained(\n        args.model_name_or_path,\n        label2id=label2id,\n        id2label=id2label,\n        ignore_mismatched_sizes=True,\n        token=args.token,\n    )\n\n    image_processor = AutoImageProcessor.from_pretrained(\n        args.model_name_or_path,\n        do_resize=True,\n        size={\"height\": args.image_height, \"width\": args.image_width},\n        do_reduce_labels=args.do_reduce_labels,\n        reduce_labels=args.do_reduce_labels,  # TODO: remove when mask2former support `do_reduce_labels`\n        token=args.token,\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define image augmentations and dataset transforms\n    # ------------------------------------------------------------------------------------------------\n    train_augment_and_transform = A.Compose(\n        [\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.5),\n            A.HueSaturationValue(p=0.1),\n        ],\n    )\n    validation_transform = A.Compose(\n        [A.NoOp()],\n    )\n\n    # Make transform functions for batch and apply for dataset splits\n    train_transform_batch = partial(\n        augment_and_transform_batch, transform=train_augment_and_transform, image_processor=image_processor\n    )\n    validation_transform_batch = partial(\n        augment_and_transform_batch, transform=validation_transform, image_processor=image_processor\n    )\n\n    dataset[\"train\"] = dataset[\"train\"].with_transform(train_transform_batch)\n    dataset[\"validation\"] = dataset[\"validation\"].with_transform(validation_transform_batch)\n\n    # ------------------------------------------------------------------------------------------------\n    # Model training and evaluation with Trainer API\n    # ------------------------------------------------------------------------------------------------\n\n    compute_metrics = Evaluator(image_processor=image_processor, id2label=id2label, threshold=0.0)\n\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=dataset[\"train\"] if training_args.do_train else None,\n        eval_dataset=dataset[\"validation\"] if training_args.do_eval else None,\n        processing_class=image_processor,\n        data_collator=collate_fn,\n        compute_metrics=compute_metrics,\n    )\n\n    # Training\n    if training_args.do_train:\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # Final evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate(eval_dataset=dataset[\"validation\"], metric_key_prefix=\"test\")\n        trainer.log_metrics(\"test\", metrics)\n        trainer.save_metrics(\"test\", metrics)\n\n    # Write model card and (optionally) push to hub\n    kwargs = {\n        \"finetuned_from\": args.model_name_or_path,\n        \"dataset\": args.dataset_name,\n        \"tags\": [\"image-segmentation\", \"instance-segmentation\", \"vision\"],\n    }\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\n**Main Functionality**\n\nThis is the main entry point of an instance segmentation model training script. It parses command-line arguments, sets up logging and telemetry, loads pre-trained model, dataset, image processor, and augmentation pipeline. The script then uses the Hugging Face Trainer API to train and evaluate the model.\n\n**Key Implementation Details**\n\n* Uses Hugging Face Transformers library for transforming arguments\n* Utilizes AutoModelForUniversalSegmentation and AutoImageProcessor for loading pre-trained model and image processor\n* Defines custom dataset transforms using A.Compose and A.Partial for horizontal flipping, random brightness contrast, and color jittering\n* Implements custom Trainer API configuration with Evaluator compute metrics\n\n**Notable Dependencies or Requirements**\n\n* Hugging Face Transformers library (`transformers`)\n* AutoModelForUniversalSegmentation and AutoImageProcessor from Hugging Face Transformers (`~ transformers.models.universal_segmentation`)\n* A.Compose and A.Partial from Hugging Face Transformers (`~transformers.pipeline`)\n* Evaluator compute metrics plugin (`@transformers.plugins.trainer.evaluators`)\n* Trainer API plugin (`@transformers.trainers`)\n* `json` library for parsing command-line arguments\n\n**Class/Function/Variable Names**\n\n* `Arguments` and `TrainingArguments` from Hugging Face Transformers\n* `parser`, `args`, and `training_args`\n* `model`, `image_processor`, `dataset`, and `trainer`\n* `compute_metrics` using the Evaluator plugin\n* `collate_fn` for dataset collator\n\n**Interfaces**\n\n* Implements Trainer API configuration with custom plugins for model training and evaluation\n* Inheriting from `Trainer` API trainer class"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:62-94

metadata:
{"docstring": "# Copied from examples/pytorch/object-detection/run_object_detection.format_image_annotations_as_coco", "end_line": 94, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "format_image_annotations_as_coco", "start_line": 62, "type": "function", "chroma:document": "CODE:\n# Copied from examples/pytorch/object-detection/run_object_detection.format_image_annotations_as_coco\ndef format_image_annotations_as_coco(\n    image_id: str, categories: List[int], areas: List[float], bboxes: List[Tuple[float]]\n) -> dict:\n    \"\"\"Format one set of image annotations to the COCO format\n\n    Args:\n        image_id (str): image id. e.g. \"0001\"\n        categories (List[int]): list of categories/class labels corresponding to provided bounding boxes\n        areas (List[float]): list of corresponding areas to provided bounding boxes\n        bboxes (List[Tuple[float]]): list of bounding boxes provided in COCO format\n            ([center_x, center_y, width, height] in absolute coordinates)\n\n    Returns:\n        dict: {\n            \"image_id\": image id,\n            \"annotations\": list of formatted annotations\n        }\n    \"\"\"\n    annotations = []\n    for category, area, bbox in zip(categories, areas, bboxes):\n        formatted_annotation = {\n            \"image_id\": image_id,\n            \"category_id\": category,\n            \"iscrowd\": 0,\n            \"area\": area,\n            \"bbox\": list(bbox),\n        }\n        annotations.append(formatted_annotation)\n\n    return {\n        \"image_id\": image_id,\n        \"annotations\": annotations,\n\n\nSUMMARY:\n**Concise Summary**\n\n### Function Purpose\n\nThe `format_image_annotations_as_coco` function formats image annotations to the COCO (Object Detection Dataset Standard) format. It takes an image ID, category labels, areas, and bounding box coordinates as input and returns a dictionary with formatted annotations.\n\n### Key Implementation Details\n\n* Formats annotation data from different lists (`categories`, `areas`, `bboxes`) into a single cohesive list.\n* Utilizes the COCO format's expected fields: `image_id`, `category_id`, `area`, `bbox`, and `iscrowd`.\n* Converts bounding box coordinates to absolute COCO format.\n\n### Dependencies/Requirements\n\n* No external dependencies; relies on standard Python data types (str, int, float, list, tuple)."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:98-117

metadata:
{"docstring": "# Copied from examples/pytorch/object-detection/run_object_detection.convert_bbox_yolo_to_pascal", "end_line": 117, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "convert_bbox_yolo_to_pascal", "start_line": 98, "type": "function", "chroma:document": "CODE:\n# Copied from examples/pytorch/object-detection/run_object_detection.convert_bbox_yolo_to_pascal\ndef convert_bbox_yolo_to_pascal(boxes: torch.Tensor, image_size: Tuple[int, int]) -> torch.Tensor:\n    \"\"\"\n    Convert bounding boxes from YOLO format (x_center, y_center, width, height) in range [0, 1]\n    to Pascal VOC format (x_min, y_min, x_max, y_max) in absolute coordinates.\n\n    Args:\n        boxes (torch.Tensor): Bounding boxes in YOLO format\n        image_size (Tuple[int, int]): Image size in format (height, width)\n\n    Returns:\n        torch.Tensor: Bounding boxes in Pascal VOC format (x_min, y_min, x_max, y_max)\n    \"\"\"\n    # convert center to corners format\n    boxes = center_to_corners_format(boxes)\n\n    # convert to absolute coordinates\n    height, width = image_size\n    boxes = boxes * torch.tensor([[width, height, width, height]])\n\n\n\nSUMMARY:\n**Summary**\n\nConverts bounding boxes from YOLO format (x_center, y_center, width, height) to Pascal VOC format (x_min, y_min, x_max, y_max) in absolute coordinates.\n\n**Key Implementation Details:**\n\n* Converts center coordinates to corner coordinates using `center_to_corners_format()`\n* Scales box coordinates by image size (width and height)\n\n**Noteability:**\n\n* This function requires input bounding boxes in YOLO format (`boxes` argument)\n* Returns output bounding boxes in Pascal VOC format (`torch.Tensor`)\n\n**External Dependencies:**\n\n* None\n\n**Implementation Choose:** `center_to_corners_format()` is not shown, please provide the source implementation if you want to add it in this summary."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:121-150

metadata:
{"docstring": "# Copied from examples/pytorch/object-detection/run_object_detection.augment_and_transform_batch", "end_line": 150, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "augment_and_transform_batch", "start_line": 121, "type": "function", "chroma:document": "CODE:\n# Copied from examples/pytorch/object-detection/run_object_detection.augment_and_transform_batch\ndef augment_and_transform_batch(\n    examples: Mapping[str, Any],\n    transform: A.Compose,\n    image_processor: AutoImageProcessor,\n    return_pixel_mask: bool = False,\n) -> BatchFeature:\n    \"\"\"Apply augmentations and format annotations in COCO format for object detection task\"\"\"\n\n    images = []\n    annotations = []\n    for image_id, image, objects in zip(examples[\"image_id\"], examples[\"image\"], examples[\"objects\"]):\n        image = np.array(image.convert(\"RGB\"))\n\n        # apply augmentations\n        output = transform(image=image, bboxes=objects[\"bbox\"], category=objects[\"category\"])\n        images.append(output[\"image\"])\n\n        # format annotations in COCO format\n        formatted_annotations = format_image_annotations_as_coco(\n            image_id, output[\"category\"], objects[\"area\"], output[\"bboxes\"]\n        )\n        annotations.append(formatted_annotations)\n\n    # Apply the image processor transformations: resizing, rescaling, normalization\n    result = image_processor(images=images, annotations=annotations, return_tensors=\"pt\")\n\n    if not return_pixel_mask:\n        result.pop(\"pixel_mask\", None)\n\n\n\nSUMMARY:\n**Concise Summary**\n\n`augment_and_transform_batch` function:\nApplies augmentations, formats annotations in COCO format, and applies image processor transformations to input batch of object detection examples.\n\n**Key Implementation Details**\n\n* Uses `A.Compose` for applying augmentations\n* Formats annotations using `format_image_annotations_as_coco`\n* Applies `AutoImageProcessor` transformations (resizing, rescaling, normalization)\n* Returns transformed batch with optional pixel mask removal\n\n**Notable Dependencies or Requirements**\n\n* `transform`: `pytorchvision.transforms.A.Compose`\n* `image_processor`: `pytautocr.image_processor.AutoImageProcessor`\n* `format_image_annotations_as_coco` (not shown in code snippet)"}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:154-160

metadata:
{"docstring": "# Copied from examples/pytorch/object-detection/run_object_detection.collate_fn", "end_line": 160, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "collate_fn", "start_line": 154, "type": "function", "chroma:document": "CODE:\n# Copied from examples/pytorch/object-detection/run_object_detection.collate_fn\ndef collate_fn(batch: List[BatchFeature]) -> Mapping[str, Union[torch.Tensor, List[Any]]]:\n    data = {}\n    data[\"pixel_values\"] = torch.stack([x[\"pixel_values\"] for x in batch])\n    data[\"labels\"] = [x[\"labels\"] for x in batch]\n    if \"pixel_mask\" in batch[0]:\n        data[\"pixel_mask\"] = torch.stack([x[\"pixel_mask\"] for x in batch])\n\n\nSUMMARY:\n**Function Summary**\n\n`collate_fn`: A function that collates and normalizes input batches for object detection tasks using PyTorch.\n\n**Key Functionality**\n\n* Takes a `batch` of `BatchFeature` objects as input\n* Returns a dictionary containing:\n\t+ Pixel values: stacked horizontally ( torch.Tensor )\n\t+ Labels: concatenated into a list ( List[Any] )\n\t+ Optional pixel mask: if present, stacked horizontally ( torch.Tensor )\n\n**Implementation Details**\n\n* Uses PyTorch tensor operations for efficient computation.\n* Utilizes list comprehension and dictionary creation for concise data processing.\n\n**Dependencies**\n\n* `torch` (tensor operations)\n* `batch` and `BatchFeature` objects from `pytorch` or custom library."}


created_at: 2025-02-05 09:13:36
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:163-173

metadata:
{"docstring": "", "end_line": 173, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "nested_to_cpu", "start_line": 163, "type": "function", "chroma:document": "CODE:\n\ndef nested_to_cpu(objects):\n    \"\"\"Move nested tesnors in objects to CPU if they are on GPU\"\"\"\n    if isinstance(objects, torch.Tensor):\n        return objects.cpu()\n    elif isinstance(objects, Mapping):\n        return type(objects)({k: nested_to_cpu(v) for k, v in objects.items()})\n    elif isinstance(objects, (list, tuple)):\n        return type(objects)([nested_to_cpu(v) for v in objects])\n    elif isinstance(objects, (np.ndarray, str, int, float, bool)):\n        return objects\n\n\nSUMMARY:\n**Function Summary: `nested_to_cpu`**\n\nThe `nested_to_cpu` function recursively moves nested tensors in objects to the CPU if they are currently on the GPU.\n\n**Main Purpose and Functionality:**\nThis function handles nested data structures of various types (`torch.Tensor`, `Mapping`, `list`, `tuple`, `np.ndarray`, `str`, `int`, `float`, `bool`) by moving any tensor-like elements that are on the GPU to the CPU, preserving their values.\n\n**Implementation Details:**\n\n*   Handles recursive handling of nested data structures using type checks (`isinstance`)\n*   Uses type-specific methods for conversion and return:\n    *   For torch.Tensor: copies the tensor to CPU using `.cpu()`\n    *   For Mapping (dict): recursively calls `nested_to_cpu` on values\n    *   For list/tuple: recursively calls `nested_to_cpu` on elements\n    *   For other scalar types, returns unchanged\n\n**Dependencies and Requirements:**\n*   Uses the PyTorch library (`torch`)\n*   Assumes presence of NumPy (`np`; not explicitly used in this snippet)\n\nNote that this summary preserves class/function/variable names exactly as per your specified requirements."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:176-224

metadata:
{"docstring": "", "end_line": 224, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "evaluation_loop", "start_line": 176, "type": "function", "chroma:document": "CODE:\n\ndef evaluation_loop(\n    model: torch.nn.Module,\n    image_processor: AutoImageProcessor,\n    accelerator: Accelerator,\n    dataloader: DataLoader,\n    id2label: Mapping[int, str],\n) -> dict:\n    model.eval()\n    metric = MeanAveragePrecision(box_format=\"xyxy\", class_metrics=True)\n\n    for step, batch in enumerate(tqdm(dataloader, disable=not accelerator.is_local_main_process)):\n        with torch.no_grad():\n            outputs = model(**batch)\n\n        # For metric computation we need to collect ground truth and predicted boxes in the same format\n\n        # 1. Collect predicted boxes, classes, scores\n        # image_processor convert boxes from YOLO format to Pascal VOC format\n        # ([x_min, y_min, x_max, y_max] in absolute coordinates)\n        image_size = torch.stack([example[\"orig_size\"] for example in batch[\"labels\"]], dim=0)\n        predictions = image_processor.post_process_object_detection(outputs, threshold=0.0, target_sizes=image_size)\n        predictions = nested_to_cpu(predictions)\n\n        # 2. Collect ground truth boxes in the same format for metric computation\n        # Do the same, convert YOLO boxes to Pascal VOC format\n        target = []\n        for label in batch[\"labels\"]:\n            label = nested_to_cpu(label)\n            boxes = convert_bbox_yolo_to_pascal(label[\"boxes\"], label[\"orig_size\"])\n            labels = label[\"class_labels\"]\n            target.append({\"boxes\": boxes, \"labels\": labels})\n\n        metric.update(predictions, target)\n\n    metrics = metric.compute()\n\n    # Replace list of per class metrics with separate metric for each class\n    classes = metrics.pop(\"classes\")\n    map_per_class = metrics.pop(\"map_per_class\")\n    mar_100_per_class = metrics.pop(\"mar_100_per_class\")\n    for class_id, class_map, class_mar in zip(classes, map_per_class, mar_100_per_class):\n        class_name = id2label[class_id.item()]\n        metrics[f\"map_{class_name}\"] = class_map\n        metrics[f\"mar_100_{class_name}\"] = class_mar\n\n    # Convert metrics to float\n    metrics = {k: round(v.item(), 4) for k, v in metrics.items()}\n\n\n\nSUMMARY:\n**Function Summary**\n\n`evaluation_loop` function evaluates the performance of an object detection model using metrics such as mean average precision (MAP), map per class, and maximum area retrieved at 100th percentile. The function takes in the following inputs:\n\n* `model`: the object detection model to be evaluated\n* `image_processor`: an AutoImageProcessor instance for converting between YOLO and Pascal VOC formats\n* `accelerator`: an Accelerator instance for parallelization\n* `dataloader`: a DataLoader instance containing training data\n* `id2label`: a dictionary mapping class IDs to their corresponding names\n\nThe function performs the following steps:\n\n1. Evaluates the model on the training dataset.\n2. Collects predicted boxes, classes, and scores from the model outputs.\n3. Converts YOLO format to Pascal VOC format for ground truth boxes.\n4. Updates the MeanAveragePrecision metric with predicted and ground truth data.\n5. Computes MAP per class and maximum area retrieved at 100th percentile.\n6. Rounds MAP values to four decimal places.\n\n**Key Implementation Details**\n\n* Uses `torch.nn.Module` as input model\n* Employs `AutoImageProcessor` for converting between YOLO and Pascal VOC formats\n* Utilizes `tqdm` for progress bar\n* Implements `with torch.no_grad():` block for zero gradient computation\n\n**Notable Dependencies or Requirements**\n\n* PyTorch, AutoImageProcessor, Accelerator, DataLoader, and `torchvision.transforms`\n* `torchviz` (not explicitly mentioned but implied through usage of `tqdm`)"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:227-389

metadata:
{"docstring": "", "end_line": 389, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "parse_args", "start_line": 227, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model for object detection task\")\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to a pretrained model or model identifier from huggingface.co/models.\",\n        default=\"facebook/detr-resnet-50\",\n    )\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        help=\"Name of the dataset on the hub.\",\n        default=\"cppe-5\",\n    )\n    parser.add_argument(\n        \"--train_val_split\",\n        type=float,\n        default=0.15,\n        help=\"Fraction of the dataset to be used for validation.\",\n    )\n    parser.add_argument(\n        \"--ignore_mismatched_sizes\",\n        action=\"store_true\",\n        help=\"Ignore mismatched sizes between the model and the dataset.\",\n    )\n    parser.add_argument(\n        \"--image_square_size\",\n        type=int,\n        default=1333,\n        help=\"Image longest size will be resized to this value, then image will be padded to square.\",\n    )\n    parser.add_argument(\n        \"--cache_dir\",\n        type=str,\n        help=\"Path to a folder in which the model and dataset will be cached.\",\n    )\n    parser.add_argument(\n        \"--use_auth_token\",\n        action=\"store_true\",\n        help=\"Whether to use an authentication token to access the model repository.\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--dataloader_num_workers\",\n        type=int,\n        default=4,\n        help=\"Number of workers to use for the dataloaders.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\n        \"--adam_beta1\",\n        type=float,\n        default=0.9,\n        help=\"Beta1 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_beta2\",\n        type=float,\n        default=0.999,\n        help=\"Beta2 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_epsilon\",\n        type=float,\n        default=1e-8,\n        help=\"Epsilon for AdamW optimizer\",\n    )\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        required=False,\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n    if args.push_to_hub or args.with_tracking:\n        if args.output_dir is None:\n            raise ValueError(\n                \"Need an `output_dir` to create a repo when `--push_to_hub` or `with_tracking` is specified.\"\n            )\n\n    if args.output_dir is not None:\n        os.makedirs(args.output_dir, exist_ok=True)\n\n\n\nSUMMARY:\n**Summary: Command Line Argument Parser**\n\nThis function defines a command line argument parser for training and fine-tuning object detection models using Hugging Face's Transformers library.\n\n**Main Purpose:**\nThe parser is used to gather configuration parameters for the model, dataset, and training process.\n\n**Key Implementation Details:**\n\n* Supports various hyperparameters for model architecture (e.g., `model_name_or_path`, `dataset_name`)\n* Handles training data preparation (e.g., `train_val_split`, `ignore_mismatched_sizes`)\n* Configures batch sizes, learning rates, and optimizer parameters (e.g., `per_device_train_batch_size`, `learning_rate`)\n* Offers options for checkpointing, experimentation, and tracking (e.g., `checkpointing_steps`, `report_to`)\n\n**Notable Dependencies:**\n\n* `argparse`: a built-in Python library for parsing command-line arguments\n* `huggingface transformers`: the library used for object detection and fine-tuning models\n\n**External Requirements:**\nNone mentioned in this code snippet, assume basic Python environment."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py:392-777

metadata:
{"docstring": "", "end_line": 777, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection_no_trainer.py", "name": "main", "start_line": 392, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_object_detection_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    # We set device_specific to True as we want different data augmentation per device.\n    if args.seed is not None:\n        set_seed(args.seed, device_specific=True)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Load dataset\n    # In distributed training, the load_dataset function guarantees that only one local process can concurrently\n    # download the dataset.\n    dataset = load_dataset(args.dataset_name, cache_dir=args.cache_dir, trust_remote_code=args.trust_remote_code)\n\n    # If we don't have a validation split, split off a percentage of train as validation.\n    args.train_val_split = None if \"validation\" in dataset.keys() else args.train_val_split\n    if isinstance(args.train_val_split, float) and args.train_val_split > 0.0:\n        split = dataset[\"train\"].train_test_split(args.train_val_split, seed=args.seed)\n        dataset[\"train\"] = split[\"train\"]\n        dataset[\"validation\"] = split[\"test\"]\n\n    # Get dataset categories and prepare mappings for label_name <-> label_id\n    categories = dataset[\"train\"].features[\"objects\"].feature[\"category\"].names\n    id2label = dict(enumerate(categories))\n    label2id = {v: k for k, v in id2label.items()}\n\n    # ------------------------------------------------------------------------------------------------\n    # Load pretrained config, model and image processor\n    # ------------------------------------------------------------------------------------------------\n\n    common_pretrained_args = {\n        \"cache_dir\": args.cache_dir,\n        \"token\": args.hub_token,\n        \"trust_remote_code\": args.trust_remote_code,\n    }\n    config = AutoConfig.from_pretrained(\n        args.model_name_or_path, label2id=label2id, id2label=id2label, **common_pretrained_args\n    )\n    model = AutoModelForObjectDetection.from_pretrained(\n        args.model_name_or_path,\n        config=config,\n        ignore_mismatched_sizes=args.ignore_mismatched_sizes,\n        **common_pretrained_args,\n    )\n    image_processor = AutoImageProcessor.from_pretrained(\n        args.model_name_or_path,\n        do_resize=True,\n        size={\"max_height\": args.image_square_size, \"max_width\": args.image_square_size},\n        do_pad=True,\n        pad_size={\"height\": args.image_square_size, \"width\": args.image_square_size},\n        **common_pretrained_args,\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define image augmentations and dataset transforms\n    # ------------------------------------------------------------------------------------------------\n    max_size = args.image_square_size\n    train_augment_and_transform = A.Compose(\n        [\n            A.Compose(\n                [\n                    A.SmallestMaxSize(max_size=max_size, p=1.0),\n                    A.RandomSizedBBoxSafeCrop(height=max_size, width=max_size, p=1.0),\n                ],\n                p=0.2,\n            ),\n            A.OneOf(\n                [\n                    A.Blur(blur_limit=7, p=0.5),\n                    A.MotionBlur(blur_limit=7, p=0.5),\n                    A.Defocus(radius=(1, 5), alias_blur=(0.1, 0.25), p=0.1),\n                ],\n                p=0.1,\n            ),\n            A.Perspective(p=0.1),\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.5),\n            A.HueSaturationValue(p=0.1),\n        ],\n        bbox_params=A.BboxParams(format=\"coco\", label_fields=[\"category\"], clip=True, min_area=25),\n    )\n    validation_transform = A.Compose(\n        [A.NoOp()],\n        bbox_params=A.BboxParams(format=\"coco\", label_fields=[\"category\"], clip=True),\n    )\n\n    # Make transform functions for batch and apply for dataset splits\n    train_transform_batch = partial(\n        augment_and_transform_batch, transform=train_augment_and_transform, image_processor=image_processor\n    )\n    validation_transform_batch = partial(\n        augment_and_transform_batch, transform=validation_transform, image_processor=image_processor\n    )\n\n    with accelerator.main_process_first():\n        train_dataset = dataset[\"train\"].with_transform(train_transform_batch)\n        valid_dataset = dataset[\"validation\"].with_transform(validation_transform_batch)\n        test_dataset = dataset[\"test\"].with_transform(validation_transform_batch)\n\n    dataloader_common_args = {\n        \"num_workers\": args.dataloader_num_workers,\n        \"collate_fn\": collate_fn,\n    }\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, batch_size=args.per_device_train_batch_size, **dataloader_common_args\n    )\n    valid_dataloader = DataLoader(\n        valid_dataset, shuffle=False, batch_size=args.per_device_eval_batch_size, **dataloader_common_args\n    )\n    test_dataloader = DataLoader(\n        test_dataset, shuffle=False, batch_size=args.per_device_eval_batch_size, **dataloader_common_args\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define optimizer, scheduler and prepare everything with the accelerator\n    # ------------------------------------------------------------------------------------------------\n\n    # Optimizer\n    optimizer = torch.optim.AdamW(\n        list(model.parameters()),\n        lr=args.learning_rate,\n        betas=[args.adam_beta1, args.adam_beta2],\n        eps=args.adam_epsilon,\n    )\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, valid_dataloader, test_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, valid_dataloader, test_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"object_detection_no_trainer\", experiment_config)\n\n    # ------------------------------------------------------------------------------------------------\n    # Run training with evaluation on each epoch\n    # ------------------------------------------------------------------------------------------------\n\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n                    if args.push_to_hub and epoch < args.num_train_epochs - 1:\n                        accelerator.wait_for_everyone()\n                        unwrapped_model = accelerator.unwrap_model(model)\n                        unwrapped_model.save_pretrained(\n                            args.output_dir,\n                            is_main_process=accelerator.is_main_process,\n                            save_function=accelerator.save,\n                        )\n                        if accelerator.is_main_process:\n                            image_processor.save_pretrained(args.output_dir)\n                            api.upload_folder(\n                                commit_message=f\"Training in progress epoch {epoch}\",\n                                folder_path=args.output_dir,\n                                repo_id=repo_id,\n                                repo_type=\"model\",\n                                token=args.hub_token,\n                            )\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        logger.info(\"***** Running evaluation *****\")\n        metrics = evaluation_loop(model, image_processor, accelerator, valid_dataloader, id2label)\n\n        logger.info(f\"epoch {epoch}: {metrics}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    **metrics,\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                image_processor.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    # ------------------------------------------------------------------------------------------------\n    # Run evaluation on test dataset and save the model\n    # ------------------------------------------------------------------------------------------------\n\n    logger.info(\"***** Running evaluation on test dataset *****\")\n    metrics = evaluation_loop(model, image_processor, accelerator, test_dataloader, id2label)\n    metrics = {f\"test_{k}\": v for k, v in metrics.items()}\n\n    logger.info(f\"Test metrics: {metrics}\")\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n                json.dump(metrics, f, indent=2)\n\n            image_processor.save_pretrained(args.output_dir)\n\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                    ignore_patterns=[\"epoch_*\"],\n\n\nSUMMARY:\nThis code is a training loop for a deep learning model, specifically an object detection model. The code uses the Hugging Face Transformers library and its integration with PyTorch. It trains the model on a specified dataset using a custom evaluation metric.\n\nThe main components of this code are:\n\n1.  **Model Training Loop**: This section contains the core training loop where the model is trained on a given batch of data.\n2.  **Evaluation Metric**: The `evaluation_loop` function calculates the evaluation metrics for both training and test datasets.\n3.  **Checkpointing**: The model saves its state at regular intervals during training, which helps in resumes from anywhere in the training process.\n\nThe benefits of this code include:\n\n*   Scalability: This code can be used to train large models on various machine learning tasks.\n*   Flexibility: The use of PyTorch and Hugging Face Transformers makes it easy to adapt the model to different datasets and tasks.\n*   Tracking Progress: The `with_tracking` flag helps track progress during training, and it will save the final results after every full epoch.\n\nHowever, there are areas for improvement:\n\n*   **Code Duplication**: Some parts of the code can be reused or extracted into separate functions for better readability.\n*   **Variable Length Loops**: Some loops in the code are not using all elements from a list (e.g., `for i, batch in enumerate(active_dataloader) if resume_step > 0`), leading to unnecessary iterations over zero-valued data. Instead, these conditions can be handled outside the loop or inside using an `if-else` statement.\n*   **Magic Numbers**: There are magic numbers scattered throughout the code (e.g., `args.max_train_steps`, `/len(train_dataloader)` etc.). It would be better to replace them with named constants for clarity and maintainability.\n\nOverall, this is a solid piece of code that effectively trains an object detection model using Hugging Face Transformers. However, refactoring and improving readability can lead to an even more efficient and well-structured version."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py:61-93

metadata:
{"docstring": "", "end_line": 93, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py", "name": "format_image_annotations_as_coco", "start_line": 61, "type": "function", "chroma:document": "CODE:\n\ndef format_image_annotations_as_coco(\n    image_id: str, categories: List[int], areas: List[float], bboxes: List[Tuple[float]]\n) -> dict:\n    \"\"\"Format one set of image annotations to the COCO format\n\n    Args:\n        image_id (str): image id. e.g. \"0001\"\n        categories (List[int]): list of categories/class labels corresponding to provided bounding boxes\n        areas (List[float]): list of corresponding areas to provided bounding boxes\n        bboxes (List[Tuple[float]]): list of bounding boxes provided in COCO format\n            ([center_x, center_y, width, height] in absolute coordinates)\n\n    Returns:\n        dict: {\n            \"image_id\": image id,\n            \"annotations\": list of formatted annotations\n        }\n    \"\"\"\n    annotations = []\n    for category, area, bbox in zip(categories, areas, bboxes):\n        formatted_annotation = {\n            \"image_id\": image_id,\n            \"category_id\": category,\n            \"iscrowd\": 0,\n            \"area\": area,\n            \"bbox\": list(bbox),\n        }\n        annotations.append(formatted_annotation)\n\n    return {\n        \"image_id\": image_id,\n        \"annotations\": annotations,\n\n\nSUMMARY:\n**Summary:**\n\n### `format_image_annotations_as_coco` function:\n\n* Formats a set of image annotations to the COCO format, accepting `image_id`, categories, areas, and bounding box coordinates.\n\n#### Primary Purpose:\nConverts annotated image data into the COCO (Visual Object Shapes) format.\n\n#### Key Implementation Details:\n\n* Iterates over `categories`, `areas`, and `bboxes` in parallel using `zip`.\n* Formats each annotation as a dictionary with required fields (`image_id`, `category_id`, `iscrowd`, `area`, `bbox`) according to the COCO format.\n* Appends formatted annotations to the `annotations` list.\n\n#### Dependencies:\n\n* None\n\n#### Notable Method Signatures and Interfaces:\n- None\n\n#### External Dependencies:\n- No external dependencies listed.\n\n### Returned Data Structure: \n\n* A dictionary with two keys:\n  - `\"image_id\"` (str): image id.\n  - `\"annotations\"` (list of dict): list of formatted annotations in COCO format."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py:96-115

metadata:
{"docstring": "", "end_line": 115, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py", "name": "convert_bbox_yolo_to_pascal", "start_line": 96, "type": "function", "chroma:document": "CODE:\n\ndef convert_bbox_yolo_to_pascal(boxes: torch.Tensor, image_size: Tuple[int, int]) -> torch.Tensor:\n    \"\"\"\n    Convert bounding boxes from YOLO format (x_center, y_center, width, height) in range [0, 1]\n    to Pascal VOC format (x_min, y_min, x_max, y_max) in absolute coordinates.\n\n    Args:\n        boxes (torch.Tensor): Bounding boxes in YOLO format\n        image_size (Tuple[int, int]): Image size in format (height, width)\n\n    Returns:\n        torch.Tensor: Bounding boxes in Pascal VOC format (x_min, y_min, x_max, y_max)\n    \"\"\"\n    # convert center to corners format\n    boxes = center_to_corners_format(boxes)\n\n    # convert to absolute coordinates\n    height, width = image_size\n    boxes = boxes * torch.tensor([[width, height, width, height]])\n\n\n\nSUMMARY:\n**Summary:**\n\n```python\n# Function: convert_bbox_yolo_to_pascal\n# Purpose: Convert YOLO format bounding boxes to Pascal VOC format in absolute coordinates.\n\ndef convert_bbox_yolo_to_pascal(boxes: torch.Tensor, image_size: Tuple[int, int]) -> torch.Tensor:\n    # Key function:\n    #   1. `center_to_corners_format`: converts center coordinates to corner form\n    #   2. Multiply by tensor containing image size values (width, height) to convert relative coordinates to absolute\n\n    # Parameters:\n    #   boxes: Bounding boxes in YOLO format (torch.Tensor)\n    #   image_size: Image size in format (height, width)\n\n    # Returns:\n    #   torch.Tensor: Converted bounding boxes in Pascal VOC format (x_min, y_min, x_max, y_max)\n```\n\n**Key Implementation Details:**\n\n* Uses `center_to_corners_format` to convert center coordinates to corner form\n* Multiplies by a tensor containing image size values to convert relative coordinates to absolute\n* Inverts box coordinates due to the YOLO format using multiplication with a tensor of width and height"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py:118-147

metadata:
{"docstring": "", "end_line": 147, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py", "name": "augment_and_transform_batch", "start_line": 118, "type": "function", "chroma:document": "CODE:\n\ndef augment_and_transform_batch(\n    examples: Mapping[str, Any],\n    transform: A.Compose,\n    image_processor: AutoImageProcessor,\n    return_pixel_mask: bool = False,\n) -> BatchFeature:\n    \"\"\"Apply augmentations and format annotations in COCO format for object detection task\"\"\"\n\n    images = []\n    annotations = []\n    for image_id, image, objects in zip(examples[\"image_id\"], examples[\"image\"], examples[\"objects\"]):\n        image = np.array(image.convert(\"RGB\"))\n\n        # apply augmentations\n        output = transform(image=image, bboxes=objects[\"bbox\"], category=objects[\"category\"])\n        images.append(output[\"image\"])\n\n        # format annotations in COCO format\n        formatted_annotations = format_image_annotations_as_coco(\n            image_id, output[\"category\"], objects[\"area\"], output[\"bboxes\"]\n        )\n        annotations.append(formatted_annotations)\n\n    # Apply the image processor transformations: resizing, rescaling, normalization\n    result = image_processor(images=images, annotations=annotations, return_tensors=\"pt\")\n\n    if not return_pixel_mask:\n        result.pop(\"pixel_mask\", None)\n\n\n\nSUMMARY:\n**Function Summary**\n\n`augment_and_transform_batch`\n\nApplies augmentations and formats annotations in COCO format for object detection tasks.\n\n* **Purpose**: Transforms input data by applying augmentations, formatting annotations, and processing images through an image processor.\n\n**Key Implementation Details**\n\n* Applies augmentations using `A.Compose`\n* Formats annotations in COCO format using `format_image_annotations_as_coco`\n* Processes images with image processor transformations (resizing, rescaling, normalization) using `AutoImageProcessor`\n\n**Notable Dependencies**\n\n* `A.Compose` for augmentation\n* `format_image_annotations_as_coco` for annotation formatting\n* `AutoImageProcessor` for image processing\n* COCO format annotations\n\n**Method Signatures and Interfaces**\n\n* `transform`: Takes `image`, `bboxes`, and `category` as inputs and returns a transformed output\n\n\nThis summary captures the primary purpose, key implementation details, and notable dependencies of the function while preserving exact class/function/variable names."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py:150-156

metadata:
{"docstring": "", "end_line": 156, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py", "name": "collate_fn", "start_line": 150, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(batch: List[BatchFeature]) -> Mapping[str, Union[torch.Tensor, List[Any]]]:\n    data = {}\n    data[\"pixel_values\"] = torch.stack([x[\"pixel_values\"] for x in batch])\n    data[\"labels\"] = [x[\"labels\"] for x in batch]\n    if \"pixel_mask\" in batch[0]:\n        data[\"pixel_mask\"] = torch.stack([x[\"pixel_mask\"] for x in batch])\n\n\nSUMMARY:\n**Summary**\n\nA custom collate function (`collate_fn`) that aggregates batches of `BatchFeature` objects into a single dictionary. It combines pixel values, labels, and optional pixel masks using PyTorch stacks.\n\n* **Primary purpose**: Aggregate batch data into a format suitable for training models.\n* **Key implementation details**:\n\t+ Stacks tensor-valued fields (`pixel_values`, `labels`) using PyTorch's `stack` function.\n\t+ Optionally stacks pixel mask tensors if present in the first batch element.\n* **Notable dependencies**: None.\n* **External dependencies**: None.\n\n**Detailed summary (more concise)**\n\n`collate_fn(batch: List[BatchFeature]) -> Mapping[str, Union[torch.Tensor, List[Any]]]`\n\nCombine batch data into a single dictionary with stacked tensors and lists."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py:323-518

metadata:
{"docstring": "", "end_line": 518, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/object-detection/run_object_detection.py", "name": "main", "start_line": 323, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_object_detection\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    checkpoint = None\n    if training_args.resume_from_checkpoint is not None:\n        checkpoint = training_args.resume_from_checkpoint\n    elif os.path.isdir(training_args.output_dir) and not training_args.overwrite_output_dir:\n        checkpoint = get_last_checkpoint(training_args.output_dir)\n        if checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # ------------------------------------------------------------------------------------------------\n    # Load dataset, prepare splits\n    # ------------------------------------------------------------------------------------------------\n\n    dataset = load_dataset(\n        data_args.dataset_name, cache_dir=model_args.cache_dir, trust_remote_code=model_args.trust_remote_code\n    )\n\n    # If we don't have a validation split, split off a percentage of train as validation\n    data_args.train_val_split = None if \"validation\" in dataset.keys() else data_args.train_val_split\n    if isinstance(data_args.train_val_split, float) and data_args.train_val_split > 0.0:\n        split = dataset[\"train\"].train_test_split(data_args.train_val_split, seed=training_args.seed)\n        dataset[\"train\"] = split[\"train\"]\n        dataset[\"validation\"] = split[\"test\"]\n\n    # Get dataset categories and prepare mappings for label_name <-> label_id\n    categories = dataset[\"train\"].features[\"objects\"].feature[\"category\"].names\n    id2label = dict(enumerate(categories))\n    label2id = {v: k for k, v in id2label.items()}\n\n    # ------------------------------------------------------------------------------------------------\n    # Load pretrained config, model and image processor\n    # ------------------------------------------------------------------------------------------------\n\n    common_pretrained_args = {\n        \"cache_dir\": model_args.cache_dir,\n        \"revision\": model_args.model_revision,\n        \"token\": model_args.token,\n        \"trust_remote_code\": model_args.trust_remote_code,\n    }\n    config = AutoConfig.from_pretrained(\n        model_args.config_name or model_args.model_name_or_path,\n        label2id=label2id,\n        id2label=id2label,\n        **common_pretrained_args,\n    )\n    model = AutoModelForObjectDetection.from_pretrained(\n        model_args.model_name_or_path,\n        config=config,\n        ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,\n        **common_pretrained_args,\n    )\n    image_processor = AutoImageProcessor.from_pretrained(\n        model_args.image_processor_name or model_args.model_name_or_path,\n        do_resize=True,\n        size={\"max_height\": data_args.image_square_size, \"max_width\": data_args.image_square_size},\n        do_pad=True,\n        pad_size={\"height\": data_args.image_square_size, \"width\": data_args.image_square_size},\n        **common_pretrained_args,\n    )\n\n    # ------------------------------------------------------------------------------------------------\n    # Define image augmentations and dataset transforms\n    # ------------------------------------------------------------------------------------------------\n    max_size = data_args.image_square_size\n    train_augment_and_transform = A.Compose(\n        [\n            A.Compose(\n                [\n                    A.SmallestMaxSize(max_size=max_size, p=1.0),\n                    A.RandomSizedBBoxSafeCrop(height=max_size, width=max_size, p=1.0),\n                ],\n                p=0.2,\n            ),\n            A.OneOf(\n                [\n                    A.Blur(blur_limit=7, p=0.5),\n                    A.MotionBlur(blur_limit=7, p=0.5),\n                    A.Defocus(radius=(1, 5), alias_blur=(0.1, 0.25), p=0.1),\n                ],\n                p=0.1,\n            ),\n            A.Perspective(p=0.1),\n            A.HorizontalFlip(p=0.5),\n            A.RandomBrightnessContrast(p=0.5),\n            A.HueSaturationValue(p=0.1),\n        ],\n        bbox_params=A.BboxParams(format=\"coco\", label_fields=[\"category\"], clip=True, min_area=25),\n    )\n    validation_transform = A.Compose(\n        [A.NoOp()],\n        bbox_params=A.BboxParams(format=\"coco\", label_fields=[\"category\"], clip=True),\n    )\n\n    # Make transform functions for batch and apply for dataset splits\n    train_transform_batch = partial(\n        augment_and_transform_batch, transform=train_augment_and_transform, image_processor=image_processor\n    )\n    validation_transform_batch = partial(\n        augment_and_transform_batch, transform=validation_transform, image_processor=image_processor\n    )\n\n    dataset[\"train\"] = dataset[\"train\"].with_transform(train_transform_batch)\n    dataset[\"validation\"] = dataset[\"validation\"].with_transform(validation_transform_batch)\n    dataset[\"test\"] = dataset[\"test\"].with_transform(validation_transform_batch)\n\n    # ------------------------------------------------------------------------------------------------\n    # Model training and evaluation with Trainer API\n    # ------------------------------------------------------------------------------------------------\n\n    eval_compute_metrics_fn = partial(\n        compute_metrics, image_processor=image_processor, id2label=id2label, threshold=0.0\n    )\n\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=dataset[\"train\"] if training_args.do_train else None,\n        eval_dataset=dataset[\"validation\"] if training_args.do_eval else None,\n        processing_class=image_processor,\n        data_collator=collate_fn,\n        compute_metrics=eval_compute_metrics_fn,\n    )\n\n    # Training\n    if training_args.do_train:\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # Final evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate(eval_dataset=dataset[\"test\"], metric_key_prefix=\"test\")\n        trainer.log_metrics(\"test\", metrics)\n        trainer.save_metrics(\"test\", metrics)\n\n    # Write model card and (optionally) push to hub\n    kwargs = {\n        \"finetuned_from\": model_args.model_name_or_path,\n        \"dataset\": data_args.dataset_name,\n        \"tags\": [\"object-detection\", \"vision\"],\n    }\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\n**Main Purpose and Functionality**\n\nThis function, `main()`, initializes the training pipeline for an object detection model using the Hugging Face Transformers library. It loads pre-trained configurations, models, and image processors, defines input data preprocessing transformations, and trains the model with the Trainer API.\n\n### Key Implementation Details\n\n1. **Data Preprocessing**:\n\t* Defines image augmentations and dataset transforms using `A.Compose` and `A.OneOf`.\n\t* Applies these transformations to training, validation, and test datasets.\n2. **Model Training**:\n\t* Initializes a Trainer object with the trained model, arguments, and datasets.\n\t* Trains the model for a specified number of epochs (if `do_train` is True).\n3. **Evaluation**:\n\t* Evaluates the model on the test dataset using the Trainer API.\n\n### Key Classes and Functions\n\n1. `Trainer`: Initializes and trains the model with pre-requisites, including data collators, metrics functions, and checkpoints.\n2. `AutoConfig`, `AutoModelForObjectDetection`, `AutoImageProcessor`: Load pre-trained configurations, models, and image processors from the Hugging Face model hub.\n3. `A.Compose` and `A.OneOf`: Define custom data augmentation and transformation pipelines.\n\n### Summary of Model Training Process\n\n1. Initialize Trainer object with pre-requisite datasets and arguments.\n2. Define data transformations for training, validation, and test datasets.\n3. Apply these transformations to the dataset.\n4. Train the model using the Trainer API and specified number of epochs (if `do_train` is True).\n5. Evaluate the model on the test dataset.\n\n### Key Variables\n\n1. `config`: Pre-trained configuration object from Hugging Face model hub.\n2. `model`: Pre-trained object detection model loaded with the configuration object.\n3. `image_processor`: Pre-trained image processor used for data augmentation and transformation.\n4. `training_args`: Training arguments, including number of epochs, evaluation metrics, and checkpoint saving.\n\n### Conclusion\n\nThis function provides an end-to-end training pipeline for object detection models using Hugging Face Transformers library. The model training process is efficient, scalable, and easy to maintain due to the powerful features provided by the library."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py:51-284

metadata:
{"docstring": "", "end_line": 284, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py", "name": "parse_args", "start_line": 51, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a text classification task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_names\",\n        nargs=\"+\",\n        type=str,\n        required=True,\n        help=\"The configuration names of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_split_names\",\n        nargs=\"+\",\n        type=str,\n        required=True,\n        help=\"The names of the training data set splits to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\",\n        type=int,\n        default=None,\n        help=\"The number of processes to use for the preprocessing.\",\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--preprocessing_only\",\n        action=\"store_true\",\n        help=\"Only run the preprocessing script to be cached for future use\",\n    )\n    parser.add_argument(\n        \"--cache_dir\",\n        type=str,\n        default=None,\n        help=\"Where do you want to store the pretrained models downloaded from huggingface.co\",\n    )\n    parser.add_argument(\n        \"--validation_split_percentage\",\n        type=int,\n        default=1,\n        help=\"Percentage of training data that should be used for validation if no validation is present in dataset.\",\n    )\n    parser.add_argument(\n        \"--logging_steps\",\n        type=int,\n        default=500,\n        help=\"Number of steps between each logging\",\n    )\n    parser.add_argument(\n        \"--saving_steps\",\n        type=int,\n        default=500,\n        help=\"Number of steps between each logging\",\n    )\n    parser.add_argument(\n        \"--audio_column_name\",\n        type=str,\n        default=\"audio\",\n        help=\"Column in the dataset that contains speech file path. Defaults to 'audio'\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=True,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--train_cache_file_name\",\n        type=str,\n        default=None,\n        help=\"Path to the train cached file name\",\n    )\n    parser.add_argument(\n        \"--validation_cache_file_name\",\n        type=str,\n        default=None,\n        help=\"Path to the validation cached file name\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--gradient_checkpointing\",\n        action=\"store_true\",\n        help=\"If True, use gradient checkpointing to save memory at the expense of slower backward pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=0, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--max_gumbel_temperature\",\n        type=float,\n        default=2.0,\n        help=\"Maximum temperature for gumbel softmax.\",\n    )\n    parser.add_argument(\n        \"--min_gumbel_temperature\",\n        type=float,\n        default=0.5,\n        help=\"Minimum temperature for gumbel softmax.\",\n    )\n    parser.add_argument(\n        \"--gumbel_temperature_decay\", type=float, default=0.999995, help=\"Decay of gumbel temperature during training.\"\n    )\n    parser.add_argument(\n        \"--max_duration_in_seconds\",\n        type=float,\n        default=5.0,\n        help=\"Filter out audio files that are longer than `max_duration_in_seconds` seconds\",\n    )\n    parser.add_argument(\n        \"--min_duration_in_seconds\",\n        type=float,\n        default=3.0,\n        help=\"Filter out audio files that are shorter than `min_duration_in_seconds` seconds\",\n    )\n    parser.add_argument(\n        \"--pad_to_multiple_of\",\n        type=int,\n        default=None,\n        help=(\n            \"If set will pad the sequence to a multiple of the provided value. This is especially useful to enable the\"\n            \" use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\"\n        ),\n    )\n    parser.add_argument(\n        \"--adam_beta1\",\n        type=float,\n        default=0.9,\n        help=\"Beta1 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_beta2\",\n        type=float,\n        default=0.999,\n        help=\"Beta2 for AdamW optimizer\",\n    )\n    parser.add_argument(\n        \"--adam_epsilon\",\n        type=float,\n        default=1e-8,\n        help=\"Epsilon for AdamW optimizer\",\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--mask_time_prob\",\n        type=float,\n        default=None,\n        help=(\n            \"Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked in the\"\n            \" contrastive task. If omitted, will pull value from model config.\"\n        ),\n    )\n    parser.add_argument(\n        \"--mask_time_length\",\n        type=int,\n        default=None,\n        help=(\n            \"Length of each vector mask span to mask along the time axis in the contrastive task.\"\n            \" If omitted, will pull value from model config.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n    if args.output_dir is not None:\n        os.makedirs(args.output_dir, exist_ok=True)\n\n\n\nSUMMARY:\n**Summary**\n\nThis function defines arguments for fine-tuning a transformer model on a text classification task. The code utilizes the Hugging Face library's `argparse` to create a command-line interface (CLI) for input parameters.\n\n**Primary Purpose**\nFine-tune a pre-trained transformer model on a text classification task based on user-provided dataset and model configuration inputs.\n\n**Implementation Details**\n\n*   Command-line arguments are used to configure various aspects of the fine-tuning process, such as dataset name, model architecture, hyperparameters (e.g., learning rate, batch size), and more.\n*   The code creates a reproducible environment by setting the random seed and ensuring that the model's output directory exists for saving and pushing updates to the Model Hub.\n\n**Notable Parameters**\n\n*   `output_dir`: Path where the fine-tuned model and other artifacts will be stored.\n*   `seed`: Random seed used for reproducibility in training.\n*   `learning_rate`, `batch_size`, `gradient_accumulation_steps`: Hyperparameters controlling the optimization process.\n*   `lr_scheduler_type` : Type of learning rate scheduler applied to optimize the model's performance.\n\n**Choice of `SchedulerType`**: The available options are linear, cosine, cosine_with_restarts, polynomial, constant, and constant_with_warmup. The user can choose one of these types based on their specific needs for fine-tuning.\n\n**Gumbel Temperature Parameters**\n\n*   `max_gumbel_temperature`: Maximum temperature value applied in the contrastive task.\n*   `min_gumbel_temperature`: Minimum temperature value applied in the contrastive task.\n*   `gumbel_temperature_decay` : Decay rate of gumbel temperature during training.\n\n**Audio Features**\n\n*   `mask_time_prob`, `mask_time_length`: Parameters controlling the masking process for audio features using Gumbel softmax."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py:376-382

metadata:
{"docstring": "", "end_line": 382, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py", "name": "multiply_grads", "start_line": 376, "type": "function", "chroma:document": "CODE:\n\ndef multiply_grads(params, c):\n    \"\"\"Multiplies grads by a constant *c*.\"\"\"\n    for p in params:\n        if p.grad is not None:\n            if torch.is_tensor(c):\n                c = c.to(p.grad.device)\n\n\nSUMMARY:\n**Summary:**\n\n`multiply_grads` function multiplies gradients by a constant `c`. It iterates over `params`, checking for non-null gradients. If `c` is a tensor, it's moved to the device of each parameter's gradient, and then multiplies the gradient data with `c`.\n\nKey implementation details:\n\n* Iterates over parameters using `for p in params:`\n* Checks if each parameter has a valid gradient (`p.grad is not None`)\n* Moves constant `c` to the same device as each parameter's gradient using `to(p grad.device)`\n* Multiplies each gradient data with `c` using `mul_`\n\nNotable dependencies:\n\n* External dependency: `torch` with specific methods (`is_tensor`, `mul_`)\n\n**Note:** The function assumes that `params` is a collection of PyTorch parameters and `c` is a scalar or tensor value."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py:385-393

metadata:
{"docstring": "", "end_line": 393, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py", "name": "get_grad_norm", "start_line": 385, "type": "function", "chroma:document": "CODE:\n\ndef get_grad_norm(params, scale=1):\n    \"\"\"Compute grad norm given a gradient scale.\"\"\"\n    total_norm = 0.0\n    for p in params:\n        if p.grad is not None:\n            param_norm = (p.grad.detach().data / scale).norm(2)\n            total_norm += param_norm.item() ** 2\n    total_norm = total_norm**0.5\n\n\nSUMMARY:\n**Summary**\n\n`get_grad_norm` function calculates the L2 gradient norm (Euclidean distance) of all model parameters `params`, scaled by `scale`. It computes the squared magnitude of each parameter's gradient and sums them up, then takes the square root.\n\n**Key Points:**\n\n* Computation:\n\t+ Iterates over model parameters (`p` in params)\n\t+ Checks if parameter has a computed gradient (`if p.grad is not None:`)\n\t+ Calculates the squared L2 norm of the gradient (`(param_grad / scale).norm(2)`)\n\t+ Accumulates sum of squared norms (`total_norm += param_norm.item() ** 2`)\n* Scaling and normalization:\n\t+ Scales gradients by `scale` (default=1)\n\t+ Applies `data` attribute to detach gradient from computation graph\n* Return value:\n\t+ Final total L2 norm computed as square root of accumulated sum"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py:396-797

metadata:
{"docstring": "", "end_line": 797, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py", "name": "main", "start_line": 396, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_wav2vec2_pretraining_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    accelerator = Accelerator()\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n\n        # set up weights and biases if available\n        if is_wandb_available():\n            import wandb\n\n            wandb.init(project=args.output_dir.split(\"/\")[-1])\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub and not args.preprocessing_only:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # 1. Download and create train, validation dataset\n    # We load all dataset configuration and datset split pairs passed in\n    # ``args.dataset_config_names`` and ``args.dataset_split_names``\n    datasets_splits = []\n    for dataset_config_name, train_split_name in zip(args.dataset_config_names, args.dataset_split_names):\n        # load dataset\n        dataset_split = load_dataset(\n            args.dataset_name,\n            dataset_config_name,\n            split=train_split_name,\n            cache_dir=args.cache_dir,\n            trust_remote_code=args.trust_remote_code,\n        )\n        datasets_splits.append(dataset_split)\n\n    # Next, we concatenate all configurations and splits into a single training dataset\n    raw_datasets = DatasetDict()\n    if len(datasets_splits) > 1:\n        raw_datasets[\"train\"] = concatenate_datasets(datasets_splits).shuffle(seed=args.seed)\n    else:\n        raw_datasets[\"train\"] = datasets_splits[0]\n\n    # Take ``args.validation_split_percentage`` from the training dataset for the validation_split_percentage\n    num_validation_samples = raw_datasets[\"train\"].num_rows * args.validation_split_percentage // 100\n\n    if num_validation_samples == 0:\n        raise ValueError(\n            \"`args.validation_split_percentage` is less than a single sample \"\n            f\"for {len(raw_datasets['train'])} training samples. Increase \"\n            \"`args.num_validation_split_percentage`. \"\n        )\n\n    raw_datasets[\"validation\"] = raw_datasets[\"train\"].select(range(num_validation_samples))\n    raw_datasets[\"train\"] = raw_datasets[\"train\"].select(range(num_validation_samples, raw_datasets[\"train\"].num_rows))\n\n    # 2. Now we preprocess the datasets including loading the audio, resampling and normalization\n    # Thankfully, `datasets` takes care of automatically loading and resampling the audio,\n    # so that we just need to set the correct target sampling rate and normalize the input\n    # via the `feature_extractor`\n    feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(args.model_name_or_path)\n\n    # make sure that dataset decodes audio with correct sampling rate\n    raw_datasets = raw_datasets.cast_column(\n        args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate)\n    )\n\n    # only normalized-inputs-training is supported\n    if not feature_extractor.do_normalize:\n        raise ValueError(\n            \"Training is only supported for normalized inputs. Make sure ``feature_extractor.do_normalize == True``\"\n        )\n\n    # set max & min audio length in number of samples\n    max_length = int(args.max_duration_in_seconds * feature_extractor.sampling_rate)\n    min_length = int(args.min_duration_in_seconds * feature_extractor.sampling_rate)\n\n    def prepare_dataset(batch):\n        sample = batch[args.audio_column_name]\n\n        inputs = feature_extractor(\n            sample[\"array\"], sampling_rate=sample[\"sampling_rate\"], max_length=max_length, truncation=True\n        )\n        batch[\"input_values\"] = inputs.input_values[0]\n        batch[\"input_length\"] = len(inputs.input_values[0])\n\n        return batch\n\n    # load via mapped files via path\n    cache_file_names = None\n    if args.train_cache_file_name is not None:\n        cache_file_names = {\"train\": args.train_cache_file_name, \"validation\": args.validation_cache_file_name}\n\n    # load audio files into numpy arrays\n    with accelerator.main_process_first():\n        vectorized_datasets = raw_datasets.map(\n            prepare_dataset,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=raw_datasets[\"train\"].column_names,\n            cache_file_names=cache_file_names,\n        )\n\n        if min_length > 0.0:\n            vectorized_datasets = vectorized_datasets.filter(\n                lambda x: x > min_length,\n                num_proc=args.preprocessing_num_workers,\n                input_columns=[\"input_length\"],\n            )\n\n        vectorized_datasets = vectorized_datasets.remove_columns(\"input_length\")\n\n    # for large datasets it is advised to run the preprocessing on a\n    # single machine first with ``args.preprocessing_only`` since there will mostly likely\n    # be a timeout when running the script in distributed mode.\n    # In a second step ``args.preprocessing_only`` can then be set to `False` to load the\n    # cached dataset\n    if args.preprocessing_only:\n        return\n\n    # 3. Load model\n    config = Wav2Vec2Config.from_pretrained(args.model_name_or_path)\n\n    # pretraining is only supported for \"newer\" stable layer norm architecture\n    # apply_spec_augment has to be True, mask_feature_prob has to be 0.0\n    if not config.do_stable_layer_norm or config.feat_extract_norm != \"layer\":\n        raise ValueError(\n            \"PreTraining is only supported for ``config.do_stable_layer_norm=True`` and\"\n            \" ``config.feat_extract_norm='layer'\"\n        )\n\n    # initialize random model\n    model = Wav2Vec2ForPreTraining(config)\n\n    # Activate gradient checkpointing if needed\n    if args.gradient_checkpointing:\n        model.gradient_checkpointing_enable()\n\n    # 4. Define data collator, optimizer and scheduler\n\n    mask_time_prob = config.mask_time_prob if args.mask_time_prob is None else args.mask_time_prob\n    mask_time_length = config.mask_time_length if args.mask_time_length is None else args.mask_time_length\n\n    data_collator = DataCollatorForWav2Vec2Pretraining(\n        model=model,\n        feature_extractor=feature_extractor,\n        pad_to_multiple_of=args.pad_to_multiple_of,\n        mask_time_prob=mask_time_prob,\n        mask_time_length=mask_time_length,\n    )\n    train_dataloader = DataLoader(\n        vectorized_datasets[\"train\"],\n        shuffle=True,\n        collate_fn=data_collator,\n        batch_size=args.per_device_train_batch_size,\n    )\n    eval_dataloader = DataLoader(\n        vectorized_datasets[\"validation\"], collate_fn=data_collator, batch_size=args.per_device_eval_batch_size\n    )\n\n    # Optimizer\n    optimizer = AdamW(\n        list(model.parameters()),\n        lr=args.learning_rate,\n        betas=[args.adam_beta1, args.adam_beta2],\n        eps=args.adam_epsilon,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader\n    )\n\n    # Scheduler and math around the number of training steps.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps,\n        num_training_steps=args.max_train_steps,\n    )\n\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # 5. Train\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(vectorized_datasets['train'])}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        for step, batch in enumerate(train_dataloader):\n            # compute num of losses\n            num_losses = batch[\"mask_time_indices\"].sum()\n            sub_attention_mask = batch.pop(\"sub_attention_mask\", None)\n            sub_attention_mask = (\n                sub_attention_mask if sub_attention_mask is not None else torch.ones_like(batch[\"mask_time_indices\"])\n            )\n            percent_masked = num_losses / sub_attention_mask.sum()\n\n            # forward\n            outputs = model(**batch)\n\n            # divide loss by gradient accumulation steps since gradients\n            # are accumulated for multiple backward passes in PyTorch\n            loss = outputs.loss / args.gradient_accumulation_steps\n            accelerator.backward(loss)\n\n            # make sure that `num_losses` is summed for distributed training\n            # and average gradients over losses of all devices\n            if accelerator.state.num_processes > 1:\n                num_losses = accelerator.gather_for_metrics(num_losses).sum()\n                gradient_multiplier = accelerator.state.num_processes / num_losses\n                multiply_grads(model.module.parameters(), gradient_multiplier)\n            else:\n                multiply_grads(model.parameters(), 1 / num_losses)\n\n            # update step\n            if (step + 1) % args.gradient_accumulation_steps == 0 or step == len(train_dataloader) - 1:\n                # compute grad norm for monitoring\n                scale = (\n                    accelerator.scaler._scale.item()\n                    if hasattr(accelerator, \"scaler\") and accelerator.scaler is not None\n                    else 1\n                )\n                if accelerator.state.num_processes > 1:\n                    grad_norm = get_grad_norm(model.module.parameters(), scale)\n                else:\n                    grad_norm = get_grad_norm(model.parameters(), scale)\n\n                # update parameters\n                optimizer.step()\n                optimizer.zero_grad()\n\n                if not accelerator.optimizer_step_was_skipped:\n                    lr_scheduler.step()\n                elif accelerator.is_local_main_process:\n                    progress_bar.write(\n                        f\"Gradients have overflown - skipping update step... Updating gradient scale to {scale}...\"\n                    )\n\n                # update gumbel temperature\n                gumbel_temperature = max(\n                    args.max_gumbel_temperature * args.gumbel_temperature_decay**completed_steps,\n                    args.min_gumbel_temperature,\n                )\n                if hasattr(model, \"module\"):\n                    model.module.set_gumbel_temperature(gumbel_temperature)\n                else:\n                    model.set_gumbel_temperature(gumbel_temperature)\n\n                progress_bar.update(1)\n                completed_steps += 1\n\n            # 6. Log all results\n            if (step + 1) % (args.gradient_accumulation_steps * args.logging_steps) == 0:\n                loss.detach()\n                outputs.contrastive_loss.detach()\n                outputs.diversity_loss.detach()\n\n                if accelerator.state.num_processes > 1:\n                    loss = accelerator.gather_for_metrics(loss).sum()\n                    outputs.contrastive_loss = accelerator.gather_for_metrics(outputs.contrastive_loss).sum()\n                    outputs.diversity_loss = accelerator.gather_for_metrics(outputs.diversity_loss).sum()\n                    percent_masked = accelerator.gather_for_metrics(percent_masked).sum()\n\n                train_logs = {\n                    \"loss\": (loss * args.gradient_accumulation_steps) / num_losses,\n                    \"constrast_loss\": outputs.contrastive_loss / num_losses,\n                    \"div_loss\": outputs.diversity_loss / num_losses,\n                    \"%_mask_idx\": percent_masked / accelerator.num_processes,\n                    \"ppl\": outputs.codevector_perplexity,\n                    \"lr\": torch.tensor(optimizer.param_groups[0][\"lr\"]),\n                    \"temp\": torch.tensor(gumbel_temperature),\n                    \"grad_norm\": torch.tensor(grad_norm),\n                }\n                log_str = \"\"\n                for k, v in train_logs.items():\n                    log_str += \"| {}: {:.3e}\".format(k, v.item())\n\n                if accelerator.is_local_main_process:\n                    progress_bar.write(log_str)\n                    if is_wandb_available():\n                        wandb.log(train_logs)\n\n            # save model every `args.saving_steps` steps\n            if (step + 1) % (args.gradient_accumulation_steps * args.saving_steps) == 0:\n                if (args.push_to_hub and epoch < args.num_train_epochs - 1) or args.output_dir is not None:\n                    accelerator.wait_for_everyone()\n                    unwrapped_model = accelerator.unwrap_model(model)\n                    unwrapped_model.save_pretrained(\n                        args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n                    )\n\n                if (args.push_to_hub and epoch < args.num_train_epochs - 1) and accelerator.is_main_process:\n                    api.upload_folder(\n                        commit_message=f\"Training in progress epoch {epoch}\",\n                        folder_path=args.output_dir,\n                        repo_id=repo_id,\n                        repo_type=\"model\",\n                        token=args.hub_token,\n                    )\n\n            # if completed steps > `args.max_train_steps` stop\n            if completed_steps >= args.max_train_steps:\n                break\n\n        # 7. Validate!\n        model.eval()\n\n        # init logs\n        val_logs = {\n            \"val_loss\": 0,\n            \"val_contrastive_loss\": 0,\n            \"val_diversity_loss\": 0,\n            \"val_num_losses\": 0,\n        }\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                batch.pop(\"sub_attention_mask\", None)\n                outputs = model(**batch)\n\n            val_logs[\"val_loss\"] += outputs.loss\n            val_logs[\"val_contrastive_loss\"] += outputs.contrastive_loss\n            val_logs[\"val_diversity_loss\"] += outputs.diversity_loss\n            val_logs[\"val_num_losses\"] += batch[\"mask_time_indices\"].sum()\n\n        # sum over devices in multi-processing\n        if accelerator.num_processes > 1:\n            val_logs = {k: accelerator.gather_for_metrics(v).sum() for k, v in val_logs.items()}\n\n        val_logs = {k: v / val_logs[\"val_num_losses\"] for k, v in val_logs.items()}\n\n        log_str = \"\"\n        for k, v in val_logs.items():\n            log_str += \"| {}: {:.3e}\".format(k, v.item())\n\n        if accelerator.is_local_main_process:\n            progress_bar.write(log_str)\n            if is_wandb_available():\n                wandb.log(val_logs)\n\n        if args.output_dir is not None:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                if args.push_to_hub:\n                    api.upload_folder(\n                        commit_message=\"End of training\",\n                        folder_path=args.output_dir,\n                        repo_id=repo_id,\n                        repo_type=\"model\",\n                        token=args.hub_token,\n\n\nSUMMARY:\nHere's a concise summary:\n\nThis code trains a model for contrastive learning and diversity loss using PyTorch. It uses multi-processing to parallelize training and validation tasks. The training process involves:\n\n1. Initialization of the model, optimizer, and scheduler\n2. Training loop:\n\t* Forward pass: calculate losses (contrastive loss, diversity loss)\n\t* Backward pass: update parameters\n3. Logging losses and metrics at specified intervals\n4. Saving the model periodically\n5. Validation and logging results\n\nThe code uses various techniques such as:\n\n1. Multi-processing to parallelize training and validation tasks\n2. Contrastive learning for self-supervised learning\n3. Diversity loss for generating new features\n4. Gradient clipping and weight decay for regularization\n5. Learning rate scheduling using a scheduler\n\nOverall, this code is designed to train a contrastive learning model with diversity, which can be used for various applications such as image classification, object detection, and retrieval tasks."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation_no_trainer.py:70-311

metadata:
{"docstring": "# Parsing input arguments", "end_line": 311, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation_no_trainer.py", "name": "parse_args", "start_line": 70, "type": "function", "chroma:document": "CODE:\n# Parsing input arguments\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a text classification task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--predict_with_generate\",\n        type=bool,\n        default=True,\n        help=\"\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n\n    parser.add_argument(\n        \"--num_beams\",\n        type=int,\n        default=None,\n        help=(\n            \"Number of beams to use for evaluation. This argument will be \"\n            \"passed to ``model.generate``, which is used during ``evaluate`` and ``predict``.\"\n        ),\n    )\n\n    parser.add_argument(\n        \"--max_source_length\",\n        type=int,\n        default=1024,\n        help=(\n            \"The maximum total input sequence length after \"\n            \"tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.\"\n        ),\n    )\n    parser.add_argument(\n        \"--max_target_length\",\n        type=int,\n        default=128,\n        help=(\n            \"The maximum total sequence length for target text after \"\n            \"tokenization. Sequences longer than this will be truncated, sequences shorter will be padded \"\n            \"during ``evaluate`` and ``predict``.\"\n        ),\n    )\n    parser.add_argument(\n        \"--val_max_target_length\",\n        type=int,\n        default=None,\n        help=(\n            \"The maximum total sequence length for validation \"\n            \"target text after tokenization.Sequences longer than this will be truncated, sequences shorter will be \"\n            \"padded. Will default to `max_target_length`.This argument is also used to override the ``max_length`` \"\n            \"param of ``model.generate``, which is used during ``evaluate`` and ``predict``.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        type=bool,\n        default=False,\n        help=(\n            \"Whether to pad all samples to model maximum sentence \"\n            \"length. If False, will pad the samples dynamically when batching to the maximum length in the batch. More \"\n            \"efficient on GPU but very bad for TPU.\"\n        ),\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--ignore_pad_token_for_loss\",\n        type=bool,\n        default=True,\n        help=\"Whether to ignore the tokens corresponding to padded labels in the loss computation or not.\",\n    )\n    parser.add_argument(\"--source_lang\", type=str, default=None, help=\"Source language id for translation.\")\n    parser.add_argument(\"--target_lang\", type=str, default=None, help=\"Target language id for translation.\")\n    parser.add_argument(\n        \"--source_prefix\",\n        type=str,\n        default=None,\n        help=\"A prefix to add before every source text (useful for T5 models).\",\n    )\n    parser.add_argument(\n        \"--preprocessing_num_workers\",\n        type=int,\n        default=None,\n        help=\"The number of processes to use for the preprocessing.\",\n    )\n    parser.add_argument(\n        \"--overwrite_cache\", action=\"store_true\", help=\"Overwrite the cached training and evaluation sets\"\n    )\n    parser.add_argument(\n        \"--max_length\",\n        type=int,\n        default=128,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,\"\n            \" sequences shorter will be padded if `--pad_to_max_length` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    # Sanity checks\n\n    if args.dataset_name is None and args.train_file is None and args.validation_file is None:\n        raise ValueError(\"Need either a task name or a training/validation file.\")\n\n    if args.train_file is not None:\n        extension = args.train_file.split(\".\")[-1]\n        assert extension in [\"csv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n    if args.validation_file is not None:\n        extension = args.validation_file.split(\".\")[-1]\n        assert extension in [\"csv\", \"json\"], \"`validation_file` should be a csv or a json file.\"\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\nThe provided code chunk is for parsing command-line arguments in Python using the `argparse` library. It defines a function called `parse_args()` that returns an object containing all the parsed arguments.\n\nHere's a breakdown of what this code does:\n\n1.  **Creating a parser**: The script starts by creating a new `ArgumentParser` named `parser`.\n\n2.  **Defining arguments**: It then adds various types of arguments to the parser using methods like `add_argument()` and `parser.add_argument_type()`. These include string, integer, float, boolean, and custom data types for scheduler and model type.\n\n3.  **Parsing arguments**: After defining all the possible arguments, the script uses the `parse_args()` method to parse these arguments from the command line and store them in a variable named `args`.\n\n4.  **Sanity checks**: It performs several sanity checks on the parsed arguments, including:\n    *   Raised an error if `dataset_name`, `train_file`, or `validation_file` is not provided.\n    *   Ensured that the file extensions of `train_file` and `validation_file` are either `.csv` or `.json`.\n    *   Raised an assertion if `output_dir` is not provided when `--push_to_hub` is used.\n\n5.  **Returning parsed arguments**: Finally, the function returns the object containing all the parsed arguments (`args`).\n\n**Example Usage**\n\nTo use this function to parse command-line arguments, you can call it like so:\n\n```python\nimport argparse\n\ndef main():\n    # Parse arguments\n    args = parser.parse_args()\n\n    # Access parsed arguments as a dictionary.\n    output_dir = args.output_dir\n    num_train_epochs = args.num_train_epochs\n    learning_rate = args.learning_rate\n    weight_decay = args.weight_decay\n\n# ... (rest of your code)\n\nif __name__ == \"__main__\":\n    main()\n```\n\nYou can customize the command-line interface by passing specific arguments during the script execution, such as:\n\n```python \n--output_dir models/saved_models --num_train_epochs 3 --learning_rate 5e-5 --weight_decay 0.01\n```\n\nThe output directory, number of training epochs, learning rate, and weight decay are all provided during script execution using these command-line arguments."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation_no_trainer.py:314-785

metadata:
{"docstring": "", "end_line": 785, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation_no_trainer.py", "name": "main", "start_line": 314, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # Parse the arguments\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_translation_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator = (\n        Accelerator(log_with=args.report_to, project_dir=args.output_dir) if args.with_tracking else Accelerator()\n    )\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files)\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(args.config_name, trust_remote_code=args.trust_remote_code)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForSeq2SeqLM.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForSeq2SeqLM.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    # Set decoder_start_token_id\n    if model.config.decoder_start_token_id is None and isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):\n        assert (\n            args.target_lang is not None and args.source_lang is not None\n        ), \"mBart requires --target_lang and --source_lang\"\n        if isinstance(tokenizer, MBartTokenizer):\n            model.config.decoder_start_token_id = tokenizer.lang_code_to_id[args.target_lang]\n        else:\n            model.config.decoder_start_token_id = tokenizer.convert_tokens_to_ids(args.target_lang)\n\n    if model.config.decoder_start_token_id is None:\n        raise ValueError(\"Make sure that `config.decoder_start_token_id` is correctly defined\")\n\n    prefix = args.source_prefix if args.source_prefix is not None else \"\"\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    column_names = raw_datasets[\"train\"].column_names\n\n    # For translation we set the codes of our source and target languages (only useful for mBART, the others will\n    # ignore those attributes).\n    if isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):\n        if args.source_lang is not None:\n            tokenizer.src_lang = args.source_lang\n        if args.target_lang is not None:\n            tokenizer.tgt_lang = args.target_lang\n\n    # Get the language codes for input/target.\n    source_lang = args.source_lang.split(\"_\")[0]\n    target_lang = args.target_lang.split(\"_\")[0]\n\n    padding = \"max_length\" if args.pad_to_max_length else False\n\n    # Temporarily set max_target_length for training.\n    max_target_length = args.max_target_length\n    padding = \"max_length\" if args.pad_to_max_length else False\n\n    def preprocess_function(examples):\n        inputs = [ex[source_lang] for ex in examples[\"translation\"]]\n        targets = [ex[target_lang] for ex in examples[\"translation\"]]\n        inputs = [prefix + inp for inp in inputs]\n        model_inputs = tokenizer(inputs, max_length=args.max_source_length, padding=padding, truncation=True)\n\n        # Tokenize targets with the `text_target` keyword argument\n        labels = tokenizer(text_target=targets, max_length=max_target_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    with accelerator.main_process_first():\n        processed_datasets = raw_datasets.map(\n            preprocess_function,\n            batched=True,\n            num_proc=args.preprocessing_num_workers,\n            remove_columns=column_names,\n            load_from_cache_file=not args.overwrite_cache,\n            desc=\"Running tokenizer on dataset\",\n        )\n\n    train_dataset = processed_datasets[\"train\"]\n    eval_dataset = processed_datasets[\"validation\"]\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    label_pad_token_id = -100 if args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n    if args.pad_to_max_length:\n        # If padding was already done ot max length, we use the default data collator that will just convert everything\n        # to tensors.\n        data_collator = default_data_collator\n    else:\n        # Otherwise, `DataCollatorWithPadding` will apply dynamic padding for us (by padding to the maximum length of\n        # the samples passed). When using mixed precision, we add `pad_to_multiple_of=8` to pad all tensors to multiple\n        # of 8s, which will enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\n        # For fp8, we pad to multiple of 16.\n        if accelerator.mixed_precision == \"fp8\":\n            pad_to_multiple_of = 16\n        elif accelerator.mixed_precision != \"no\":\n            pad_to_multiple_of = 8\n        else:\n            pad_to_multiple_of = None\n        data_collator = DataCollatorForSeq2Seq(\n            tokenizer,\n            model=model,\n            label_pad_token_id=label_pad_token_id,\n            pad_to_multiple_of=pad_to_multiple_of,\n        )\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\", \"layer_norm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps,\n        num_training_steps=args.max_train_steps,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # We initialize the trackers only on main process because `accelerator.log`\n    # only logs on main process and we don't want empty logs/runs on other processes.\n    if args.with_tracking:\n        if accelerator.is_main_process:\n            experiment_config = vars(args)\n            # TensorBoard cannot log Enums, need the raw value\n            experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n            accelerator.init_trackers(\"translation_no_trainer\", experiment_config)\n\n    metric = evaluate.load(\"sacrebleu\")\n\n    def postprocess_text(preds, labels):\n        preds = [pred.strip() for pred in preds]\n        labels = [[label.strip()] for label in labels]\n\n        return preds, labels\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            outputs = model(**batch)\n            loss = outputs.loss\n            # We keep track of the loss at each epoch\n            if args.with_tracking:\n                total_loss += loss.detach().float()\n            loss = loss / args.gradient_accumulation_steps\n            accelerator.backward(loss)\n            if step % args.gradient_accumulation_steps == 0 or step == len(train_dataloader) - 1:\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n\n        if args.val_max_target_length is None:\n            args.val_max_target_length = args.max_target_length\n\n        gen_kwargs = {\n            \"max_length\": args.val_max_target_length if args is not None else config.max_length,\n            \"num_beams\": args.num_beams,\n        }\n        samples_seen = 0\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                generated_tokens = accelerator.unwrap_model(model).generate(\n                    batch[\"input_ids\"],\n                    attention_mask=batch[\"attention_mask\"],\n                    **gen_kwargs,\n                )\n\n                generated_tokens = accelerator.pad_across_processes(\n                    generated_tokens, dim=1, pad_index=tokenizer.pad_token_id\n                )\n                labels = batch[\"labels\"]\n                if not args.pad_to_max_length:\n                    # If we did not pad to max length, we need to pad the labels too\n                    labels = accelerator.pad_across_processes(batch[\"labels\"], dim=1, pad_index=tokenizer.pad_token_id)\n\n                generated_tokens = accelerator.gather(generated_tokens).cpu().numpy()\n                labels = accelerator.gather(labels).cpu().numpy()\n\n                if args.ignore_pad_token_for_loss:\n                    # Replace -100 in the labels as we can't decode them.\n                    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n\n                decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n                decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n                decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n\n                # If we are in a multiprocess environment, the last batch has duplicates\n                if accelerator.num_processes > 1:\n                    if step == len(eval_dataloader) - 1:\n                        decoded_preds = decoded_preds[: len(eval_dataloader.dataset) - samples_seen]\n                        decoded_labels = decoded_labels[: len(eval_dataloader.dataset) - samples_seen]\n                    else:\n                        samples_seen += len(decoded_labels)\n\n                metric.add_batch(predictions=decoded_preds, references=decoded_labels)\n        eval_metric = metric.compute()\n        logger.info({\"bleu\": eval_metric[\"score\"]})\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"bleu\": eval_metric[\"score\"],\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n        with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n\n\nSUMMARY:\nThis code is a training script for a machine learning model, specifically a transformer-based language model. The script uses the Hugging Face Transformers library and the PyTorch framework.\n\nKey features of the script include:\n\n1. **Data loading**: The script loads data from a dataset (not specified in the provided code snippet) using Hugging Face's `DataLoader`.\n2. **Model training**: The script trains the model using PyTorch's `Trainer` API, with support for automatic mixed precision training.\n3. **Evaluation metrics**: The script uses the BLEU score as an evaluation metric to measure the model's performance on a validation set.\n4. **Checkpointing and saving**: The script saves model checkpoints at regular intervals during training using Hugging Face's `Trainer` API.\n\nThe script also includes features for:\n\n* **Multiprocessing**: The script utilizes PyTorch's multi-GPU support with multiple workers to speed up training times.\n* **Hyperparameter tuning**: Although not explicitly implemented in this code snippet, the `Trainer` API allows for hyperparameter tuning through the use of a `TrainerCallback` class.\n\nOverall, this script appears to be designed for training a transformer-based language model on a specific dataset."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation.py:260-687

metadata:
{"docstring": "", "end_line": 687, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation.py", "name": "main", "start_line": 260, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_translation\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    if data_args.source_prefix is None and model_args.model_name_or_path in [\n        \"google-t5/t5-small\",\n        \"google-t5/t5-base\",\n        \"google-t5/t5-large\",\n        \"google-t5/t5-3b\",\n        \"google-t5/t5-11b\",\n    ]:\n        logger.warning(\n            \"You're running a t5 model but didn't provide a source prefix, which is expected, e.g. with \"\n            \"`--source_prefix 'translate English to German: ' `\"\n        )\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own JSON training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For translation, only JSON files are supported, with one field named \"translation\" containing two keys for the\n    # source and target languages (unless you adapt what follows).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        if data_args.test_file is not None:\n            data_files[\"test\"] = data_args.test_file\n            extension = data_args.test_file.split(\".\")[-1]\n        if extension == \"jsonl\":\n            builder_name = \"json\"  # the \"json\" builder reads both .json and .jsonl files\n        else:\n            builder_name = extension  # e.g. \"parquet\"\n        raw_datasets = load_dataset(\n            builder_name,\n            data_files=data_files,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading.\n\n    # Load pretrained model and tokenizer\n    #\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        use_fast=model_args.use_fast_tokenizer,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForSeq2SeqLM.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    # Set decoder_start_token_id\n    if model.config.decoder_start_token_id is None and isinstance(tokenizer, (MBartTokenizer, MBartTokenizerFast)):\n        if isinstance(tokenizer, MBartTokenizer):\n            model.config.decoder_start_token_id = tokenizer.lang_code_to_id[data_args.target_lang]\n        else:\n            model.config.decoder_start_token_id = tokenizer.convert_tokens_to_ids(data_args.target_lang)\n\n    if model.config.decoder_start_token_id is None:\n        raise ValueError(\"Make sure that `config.decoder_start_token_id` is correctly defined\")\n\n    prefix = data_args.source_prefix if data_args.source_prefix is not None else \"\"\n\n    # Preprocessing the datasets.\n    # We need to tokenize inputs and targets.\n    if training_args.do_train:\n        column_names = raw_datasets[\"train\"].column_names\n    elif training_args.do_eval:\n        column_names = raw_datasets[\"validation\"].column_names\n    elif training_args.do_predict:\n        column_names = raw_datasets[\"test\"].column_names\n    else:\n        logger.info(\"There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.\")\n        return\n\n    # For translation we set the codes of our source and target languages (only useful for mBART, the others will\n    # ignore those attributes).\n    if isinstance(tokenizer, tuple(MULTILINGUAL_TOKENIZERS)):\n        assert data_args.target_lang is not None and data_args.source_lang is not None, (\n            f\"{tokenizer.__class__.__name__} is a multilingual tokenizer which requires --source_lang and \"\n            \"--target_lang arguments.\"\n        )\n\n        tokenizer.src_lang = data_args.source_lang\n        tokenizer.tgt_lang = data_args.target_lang\n\n        # For multilingual translation models like mBART-50 and M2M100 we need to force the target language token\n        # as the first generated token. We ask the user to explicitly provide this as --forced_bos_token argument.\n        forced_bos_token_id = (\n            tokenizer.lang_code_to_id[data_args.forced_bos_token] if data_args.forced_bos_token is not None else None\n        )\n        model.config.forced_bos_token_id = forced_bos_token_id\n\n    # Get the language codes for input/target.\n    source_lang = data_args.source_lang.split(\"_\")[0]\n    target_lang = data_args.target_lang.split(\"_\")[0]\n\n    # Check the whether the source target length fits in the model, if it has absolute positional embeddings\n    if (\n        hasattr(model.config, \"max_position_embeddings\")\n        and not hasattr(model.config, \"relative_attention_max_distance\")\n        and model.config.max_position_embeddings < data_args.max_source_length\n    ):\n        raise ValueError(\n            f\"`--max_source_length` is set to {data_args.max_source_length}, but the model only has\"\n            f\" {model.config.max_position_embeddings} position encodings. Consider either reducing\"\n            f\" `--max_source_length` to {model.config.max_position_embeddings} or using a model with larger position \"\n            \"embeddings\"\n        )\n\n    # Temporarily set max_target_length for training.\n    max_target_length = data_args.max_target_length\n    padding = \"max_length\" if data_args.pad_to_max_length else False\n\n    if training_args.label_smoothing_factor > 0 and not hasattr(model, \"prepare_decoder_input_ids_from_labels\"):\n        logger.warning(\n            \"label_smoothing is enabled but the `prepare_decoder_input_ids_from_labels` method is not defined for \"\n            f\"`{model.__class__.__name__}`. This will lead to loss being calculated twice and will take up more memory\"\n        )\n\n    def preprocess_function(examples):\n        inputs = [ex[source_lang] for ex in examples[\"translation\"]]\n        targets = [ex[target_lang] for ex in examples[\"translation\"]]\n        inputs = [prefix + inp for inp in inputs]\n        model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, padding=padding, truncation=True)\n\n        # Tokenize targets with the `text_target` keyword argument\n        labels = tokenizer(text_target=targets, max_length=max_target_length, padding=padding, truncation=True)\n\n        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore\n        # padding in the loss.\n        if padding == \"max_length\" and data_args.ignore_pad_token_for_loss:\n            labels[\"input_ids\"] = [\n                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels[\"input_ids\"]\n            ]\n\n        model_inputs[\"labels\"] = labels[\"input_ids\"]\n        return model_inputs\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on train dataset\",\n            )\n\n    if training_args.do_eval:\n        max_target_length = data_args.val_max_target_length\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_dataset = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on validation dataset\",\n            )\n\n    if training_args.do_predict:\n        max_target_length = data_args.val_max_target_length\n        if \"test\" not in raw_datasets:\n            raise ValueError(\"--do_predict requires a test dataset\")\n        predict_dataset = raw_datasets[\"test\"]\n        if data_args.max_predict_samples is not None:\n            max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples)\n            predict_dataset = predict_dataset.select(range(max_predict_samples))\n        with training_args.main_process_first(desc=\"prediction dataset map pre-processing\"):\n            predict_dataset = predict_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                remove_columns=column_names,\n                load_from_cache_file=not data_args.overwrite_cache,\n                desc=\"Running tokenizer on prediction dataset\",\n            )\n\n    # Data collator\n    label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id\n    if data_args.pad_to_max_length:\n        data_collator = default_data_collator\n    else:\n        data_collator = DataCollatorForSeq2Seq(\n            tokenizer,\n            model=model,\n            label_pad_token_id=label_pad_token_id,\n            pad_to_multiple_of=8 if training_args.fp16 else None,\n        )\n\n    # Metric\n    metric = evaluate.load(\"sacrebleu\", cache_dir=model_args.cache_dir)\n\n    def postprocess_text(preds, labels):\n        preds = [pred.strip() for pred in preds]\n        labels = [[label.strip()] for label in labels]\n\n        return preds, labels\n\n    def compute_metrics(eval_preds):\n        preds, labels = eval_preds\n        if isinstance(preds, tuple):\n            preds = preds[0]\n        # Replace -100s used for padding as we can't decode them\n        preds = np.where(preds != -100, preds, tokenizer.pad_token_id)\n        decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n        labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n        decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n\n        # Some simple post-processing\n        decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)\n\n        result = metric.compute(predictions=decoded_preds, references=decoded_labels)\n        result = {\"bleu\": result[\"score\"]}\n\n        prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds]\n        result[\"gen_len\"] = np.mean(prediction_lens)\n        result = {k: round(v, 4) for k, v in result.items()}\n        return result\n\n    # Initialize our Trainer\n    trainer = Seq2SeqTrainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics if training_args.predict_with_generate else None,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n\n        metrics = train_result.metrics\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    results = {}\n    max_length = (\n        training_args.generation_max_length\n        if training_args.generation_max_length is not None\n        else data_args.val_max_target_length\n    )\n    num_beams = data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n\n        metrics = trainer.evaluate(max_length=max_length, num_beams=num_beams, metric_key_prefix=\"eval\")\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    if training_args.do_predict:\n        logger.info(\"*** Predict ***\")\n\n        predict_results = trainer.predict(\n            predict_dataset, metric_key_prefix=\"predict\", max_length=max_length, num_beams=num_beams\n        )\n        metrics = predict_results.metrics\n        max_predict_samples = (\n            data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset)\n        )\n        metrics[\"predict_samples\"] = min(max_predict_samples, len(predict_dataset))\n\n        trainer.log_metrics(\"predict\", metrics)\n        trainer.save_metrics(\"predict\", metrics)\n\n        if trainer.is_world_process_zero():\n            if training_args.predict_with_generate:\n                predictions = predict_results.predictions\n                predictions = np.where(predictions != -100, predictions, tokenizer.pad_token_id)\n                predictions = tokenizer.batch_decode(\n                    predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True\n                )\n                predictions = [pred.strip() for pred in predictions]\n                output_prediction_file = os.path.join(training_args.output_dir, \"generated_predictions.txt\")\n                with open(output_prediction_file, \"w\", encoding=\"utf-8\") as writer:\n                    writer.write(\"\\n\".join(predictions))\n\n    kwargs = {\"finetuned_from\": model_args.model_name_or_path, \"tasks\": \"translation\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    languages = [l for l in [data_args.source_lang, data_args.target_lang] if l is not None]\n    if len(languages) > 0:\n        kwargs[\"language\"] = languages\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n        trainer.create_model_card(**kwargs)\n\n\n\nSUMMARY:\nThis is a code for training a translation model using the Hugging Face Transformers library. The code defines a sequence-to-sequence model, sets up the data processing pipeline, and trains the model on a provided dataset.\n\n**Key Features:**\n\n1. The code uses the `Seq2SeqTrainer` class from the transformers library to train the model.\n2. It sets up the data processing pipeline using the `tokenizer` and `processing_class`.\n3. The model is trained on the `train_dataset` with a maximum length of `max_length` and `num_beams`.\n4. Evaluation metrics (Bleu score, generation length) are computed during the training process.\n5. The code also includes options for pushing the model to Hugging Face Hub.\n\n**Key Steps:**\n\n1. Load pre-trained tokenizer\n2. Define data processing pipeline using `tokenizer` and `processing_class`\n3. Split dataset into training, validation, and prediction sets\n4. Train the model using `Seq2SeqTrainer`\n5. Evaluate the model on validation and prediction datasets\n6. Save the model and evaluate its performance using Bleu score"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation.py:690-692

metadata:
{"docstring": "", "end_line": 692, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/translation/run_translation.py", "name": "_mp_fn", "start_line": 690, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Summary:**\n\nThis Python function `_mp_fn` appears to be a part of a function scheduling system, possibly for parallelization or job management. It takes an `index` parameter and:\n- Calls the `main` function when executed by XLA Spawn (a TPU execution engine).\n- Primary purpose: Schedule main execution on TPUs.\n\n**Key Implementation Details:**\n\n* **Function Signature:** `_mp_fn(index)`\n* **Implementation Note:** Calls `main()` as part of task scheduling.\n\n**Dependencies and Requirements:**\n\n* **External Dependencies:**\n\t+ XLA Spawn (TPU execution engine)\n* **Inherited/Implemented Interfaces:** None mentioned"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/audio-classification/run_audio_classification.py:52-58

metadata:
{"docstring": "", "end_line": 58, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/audio-classification/run_audio_classification.py", "name": "random_subsample", "start_line": 52, "type": "function", "chroma:document": "CODE:\n\ndef random_subsample(wav: np.ndarray, max_length: float, sample_rate: int = 16000):\n    \"\"\"Randomly sample chunks of `max_length` seconds from the input audio\"\"\"\n    sample_length = int(round(sample_rate * max_length))\n    if len(wav) <= sample_length:\n        return wav\n    random_offset = randint(0, len(wav) - sample_length - 1)\n\n\nSUMMARY:\n**Function Summary: `random_subsample`**\n\n* **Purpose:** Randomly selects chunks of audio data from an input WAV file, limited to a specified maximum length.\n* **Functionality:** Samples up to `max_length` seconds at the specified sample rate (default: 16,000 Hz).\n* **Implementation Details:**\n\t+ Sampling is done using a random offset within the file bounds.\n\t+ The selected chunk size is determined by rounding the product of sample rate and `max_length`.\n* **Params:**\n\t+ `wav`: Input audio data (`np.ndarray`)\n\t+ `max_length`: Max chunk length in seconds\n\t+ `sample_rate` (optional): Audio sampling rate, defaults to 16,000 Hz.\n\n**Key Points:**\n\n* Uses `numpy` for array manipulation\n* Incorporates randomization using `randint` for offset selection"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/audio-classification/run_audio_classification.py:197-428

metadata:
{"docstring": "", "end_line": 428, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/audio-classification/run_audio_classification.py", "name": "main", "start_line": 197, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_audio_classification\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to train from scratch.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Initialize our dataset and prepare it for the audio classification task.\n    raw_datasets = DatasetDict()\n    raw_datasets[\"train\"] = load_dataset(\n        data_args.dataset_name,\n        data_args.dataset_config_name,\n        split=data_args.train_split_name,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    raw_datasets[\"eval\"] = load_dataset(\n        data_args.dataset_name,\n        data_args.dataset_config_name,\n        split=data_args.eval_split_name,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    if data_args.audio_column_name not in raw_datasets[\"train\"].column_names:\n        raise ValueError(\n            f\"--audio_column_name {data_args.audio_column_name} not found in dataset '{data_args.dataset_name}'. \"\n            \"Make sure to set `--audio_column_name` to the correct audio column - one of \"\n            f\"{', '.join(raw_datasets['train'].column_names)}.\"\n        )\n\n    if data_args.label_column_name not in raw_datasets[\"train\"].column_names:\n        raise ValueError(\n            f\"--label_column_name {data_args.label_column_name} not found in dataset '{data_args.dataset_name}'. \"\n            \"Make sure to set `--label_column_name` to the correct text column - one of \"\n            f\"{', '.join(raw_datasets['train'].column_names)}.\"\n        )\n\n    # Setting `return_attention_mask=True` is the way to get a correctly masked mean-pooling over\n    # transformer outputs in the classifier, but it doesn't always lead to better accuracy\n    feature_extractor = AutoFeatureExtractor.from_pretrained(\n        model_args.feature_extractor_name or model_args.model_name_or_path,\n        return_attention_mask=model_args.attention_mask,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # `datasets` takes care of automatically loading and resampling the audio,\n    # so we just need to set the correct target sampling rate.\n    raw_datasets = raw_datasets.cast_column(\n        data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate)\n    )\n\n    model_input_name = feature_extractor.model_input_names[0]\n\n    def train_transforms(batch):\n        \"\"\"Apply train_transforms across a batch.\"\"\"\n        subsampled_wavs = []\n        for audio in batch[data_args.audio_column_name]:\n            wav = random_subsample(\n                audio[\"array\"], max_length=data_args.max_length_seconds, sample_rate=feature_extractor.sampling_rate\n            )\n            subsampled_wavs.append(wav)\n        inputs = feature_extractor(subsampled_wavs, sampling_rate=feature_extractor.sampling_rate)\n        output_batch = {model_input_name: inputs.get(model_input_name)}\n        output_batch[\"labels\"] = list(batch[data_args.label_column_name])\n\n        return output_batch\n\n    def val_transforms(batch):\n        \"\"\"Apply val_transforms across a batch.\"\"\"\n        wavs = [audio[\"array\"] for audio in batch[data_args.audio_column_name]]\n        inputs = feature_extractor(wavs, sampling_rate=feature_extractor.sampling_rate)\n        output_batch = {model_input_name: inputs.get(model_input_name)}\n        output_batch[\"labels\"] = list(batch[data_args.label_column_name])\n\n        return output_batch\n\n    # Prepare label mappings.\n    # We'll include these in the model's config to get human readable labels in the Inference API.\n    labels = raw_datasets[\"train\"].features[data_args.label_column_name].names\n    label2id, id2label = {}, {}\n    for i, label in enumerate(labels):\n        label2id[label] = str(i)\n        id2label[str(i)] = label\n\n    # Load the accuracy metric from the datasets package\n    metric = evaluate.load(\"accuracy\", cache_dir=model_args.cache_dir)\n\n    # Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with\n    # `predictions` and `label_ids` fields) and has to return a dictionary string to float.\n    def compute_metrics(eval_pred):\n        \"\"\"Computes accuracy on a batch of predictions\"\"\"\n        predictions = np.argmax(eval_pred.predictions, axis=1)\n        return metric.compute(predictions=predictions, references=eval_pred.label_ids)\n\n    config = AutoConfig.from_pretrained(\n        model_args.config_name or model_args.model_name_or_path,\n        num_labels=len(labels),\n        label2id=label2id,\n        id2label=id2label,\n        finetuning_task=\"audio-classification\",\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForAudioClassification.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n        ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,\n    )\n\n    # freeze the convolutional waveform encoder\n    if model_args.freeze_feature_encoder:\n        model.freeze_feature_encoder()\n\n    if training_args.do_train:\n        if data_args.max_train_samples is not None:\n            raw_datasets[\"train\"] = (\n                raw_datasets[\"train\"].shuffle(seed=training_args.seed).select(range(data_args.max_train_samples))\n            )\n        # Set the training transforms\n        raw_datasets[\"train\"].set_transform(train_transforms, output_all_columns=False)\n\n    if training_args.do_eval:\n        if data_args.max_eval_samples is not None:\n            raw_datasets[\"eval\"] = (\n                raw_datasets[\"eval\"].shuffle(seed=training_args.seed).select(range(data_args.max_eval_samples))\n            )\n        # Set the validation transforms\n        raw_datasets[\"eval\"].set_transform(val_transforms, output_all_columns=False)\n\n    # Initialize our trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=raw_datasets[\"train\"] if training_args.do_train else None,\n        eval_dataset=raw_datasets[\"eval\"] if training_args.do_eval else None,\n        compute_metrics=compute_metrics,\n        processing_class=feature_extractor,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate()\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # Write model card and (optionally) push to hub\n    kwargs = {\n        \"finetuned_from\": model_args.model_name_or_path,\n        \"tasks\": \"audio-classification\",\n        \"dataset\": data_args.dataset_name,\n        \"tags\": [\"audio-classification\"],\n    }\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\n**Audio Classification Model Training with Transformers**\n==========================================================\n\nThis script trains an audio classification model using the Hugging Face Transformers library. The model is configured for fine-tuning on a custom dataset, with options to train and evaluate the model.\n\n**Dataset Configuration**\n------------------------\n\nThe dataset consists of:\n\n* `train` dataset: A subset of size `data_args.max_train_samples` from the full training set.\n* `eval` dataset: A subset of size `data_args.max_eval_samples` from the full evaluation set.\n* The audio feature extractor is created using the `feature_extractor` processing class.\n\n**Model Configuration**\n----------------------\n\nThe model is a pre-trained `AudioModelForSequenceClassification`. The configuration is updated to match the number of labels in the dataset, with custom label mappings generated from the dataset's label names.\n\n**Training and Evaluation**\n---------------------------\n\nThe training process involves:\n\n* Computing metrics using the `compute_metrics` function.\n* Saving the model, logs, and state.\n* Training the model using the `Trainer` class.\n* Evaluating the model using the `evaluate` method.\n\n**Key Options**\n--------------\n\n* `data_args.max_train_samples`: Maximum number of samples to use for training.\n* `data_args.max_eval_samples`: Maximum number of samples to use for evaluation.\n* `training_args.do_train`: Flag to indicate whether to train the model or not.\n* `training_args.do_eval`: Flag to indicate whether to evaluate the model or not.\n* `training_args.resume_from_checkpoint`: Path to a checkpoint to resume training.\n\n**Example Use Cases**\n---------------------\n\nThis script can be used as a starting point for audio classification tasks, such as:\n\n* Sentiment analysis of speech recordings\n* Emotion detection from music\n* Voice recognition systems\n\n**Advice and Next Steps**\n-------------------------\n\n* Experiment with different dataset sizes and architectures.\n* Try adjusting the hyperparameters to optimize performance.\n* Explore other Transformers library features, such as pipeline optimization or efficient inference."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:94-101

metadata:
{"docstring": "#\n# Functions to prepare models' input\n#", "end_line": 101, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "prepare_ctrl_input", "start_line": 94, "type": "function", "chroma:document": "CODE:\n\ndef prepare_ctrl_input(args, _, tokenizer, prompt_text):\n    if args.temperature > 0.7:\n        logger.info(\"CTRL typically works better with lower temperatures (and lower top_k).\")\n\n    encoded_prompt = tokenizer.encode(prompt_text, add_special_tokens=False)\n    if not any(encoded_prompt[0] == x for x in tokenizer.control_codes.values()):\n        logger.info(\"WARNING! You are not starting your generation from a control code so you won't get good results\")\n\n\nSUMMARY:\n**Function Summary**\n\n`prepare_ctrl_input`\n================================\n\n### Main Purpose and Functionality\n\nPrepares input for CTRL (Conversational Toolkit for Reasoning and Training) by encoding the prompt text.\n\n### Key Implementation Details\n\n* Checks temperature threshold (0.7) and logs a warning if above.\n* Encodes prompt text using `tokenizer` with `add_special_tokens=False`.\n* Verifies that encoded prompt starts with a control code; logs a warning otherwise.\n\n### Notable Dependencies or Requirements\n\n* `args`: Input arguments\n* `_`: Ignored argument (required, but not used)\n* `tokenizer`: Custom tokenizer object\n* `prompt_text`: Input text to be encoded\n\n### External Dependencies\n\n* Custom tokenizer implementation (`tokenizer`)\n* logging module (`logger`)"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:104-127

metadata:
{"docstring": "", "end_line": 127, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "prepare_xlm_input", "start_line": 104, "type": "function", "chroma:document": "CODE:\n\ndef prepare_xlm_input(args, model, tokenizer, prompt_text):\n    # kwargs = {\"language\": None, \"mask_token_id\": None}\n\n    # Set the language\n    use_lang_emb = hasattr(model.config, \"use_lang_emb\") and model.config.use_lang_emb\n    if hasattr(model.config, \"lang2id\") and use_lang_emb:\n        available_languages = model.config.lang2id.keys()\n        if args.xlm_language in available_languages:\n            language = args.xlm_language\n        else:\n            language = None\n            while language not in available_languages:\n                language = input(\"Using XLM. Select language in \" + str(list(available_languages)) + \" >>> \")\n\n        model.config.lang_id = model.config.lang2id[language]\n        # kwargs[\"language\"] = tokenizer.lang2id[language]\n\n    # TODO fix mask_token_id setup when configurations will be synchronized between models and tokenizers\n    # XLM masked-language modeling (MLM) models need masked token\n    # is_xlm_mlm = \"mlm\" in args.model_name_or_path\n    # if is_xlm_mlm:\n    #     kwargs[\"mask_token_id\"] = tokenizer.mask_token_id\n\n\n\nSUMMARY:\n**Summary:**\n\n`prepare_xlm_input` function prepares input for XLM (Extreme Language Model) based on provided arguments and model configurations.\n\n**Key Features:**\n\n* Sets language and retrieves corresponding ID from `lang2id` dictionary\n* Prompts user to select language if not present in available languages\n* Initializes `mask_token_id` when applicable (TODO)\n\n**Method Signatures and Interfaces:**\n\n* `prepare_xlm_input(args, model, tokenizer, prompt_text)`\n\n**Notable Dependencies and Requirements:**\n\n* `args.model_name_or_path`\n* `model.config.lang2id`\n* `tokenizer lang2id`\n\n**Primary Purpose:**\nThis function prepares input for XLM-based models by setting language and retrieving corresponding ID. It also prompts user to select language if not present in available languages."}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:130-133

metadata:
{"docstring": "", "end_line": 133, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "prepare_xlnet_input", "start_line": 130, "type": "function", "chroma:document": "CODE:\n\ndef prepare_xlnet_input(args, _, tokenizer, prompt_text):\n    prefix = args.prefix if args.prefix else args.padding_text if args.padding_text else PREFIX\n    prompt_text = prefix + prompt_text\n\n\nSUMMARY:\n**Concise Summary**\n\n**Functionality:** Prepares input for XLNet model by concatenating a prefixed text with the provided `prompt_text`.\n\n**Key Details:**\n- Concatenates `prefix` with `prompt_text` if available\n- Uses default values (`args.prefix`, `args.padding_text`) in absence of explicit value\n- Default prefix is specified as `PREFIX`\n\n**Notable Dependencies/Requirements:**\n- `args`: Expected dictionary containing input parameters\n- `tokenizer`\n- `_` ( Unused argument)\n- `prompt_text`"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:136-139

metadata:
{"docstring": "", "end_line": 139, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "prepare_transfoxl_input", "start_line": 136, "type": "function", "chroma:document": "CODE:\n\ndef prepare_transfoxl_input(args, _, tokenizer, prompt_text):\n    prefix = args.prefix if args.prefix else args.padding_text if args.padding_text else PREFIX\n    prompt_text = prefix + prompt_text\n\n\nSUMMARY:\n**Summary:**\n\n`prepare_transfoxl_input` function:\n\n* Prepares the input for a `transfoxl` transformer-based model.\n* Concatenates a prefix (defaulting to `args.padding_text` or `PREFIX`) with the provided `prompt_text`.\n* Returns the modified prompt text.\n\n**Key implementation details:**\n\n* Uses the `prefix` and default values from the `args` dictionary (`prefix`, `padding_text`)\n* Concatenation of `prefix` and `prompt_text`\n * Constants:\n\t+ `PREFIX`: Default prefix (used as fallback)\n\n**Notable dependencies or requirements:**\n\n* `tokenizer` object (not explicitly used in this code chunk)\n* External dependency: `_` (likely a wildcard or placeholder, its purpose not clear without more context)"}


created_at: 2025-02-05 09:13:37
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:150-157

metadata:
{"docstring": "", "end_line": 157, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "adjust_length_to_model", "start_line": 150, "type": "function", "chroma:document": "CODE:\n\ndef adjust_length_to_model(length, max_sequence_length):\n    if length < 0 and max_sequence_length > 0:\n        length = max_sequence_length\n    elif 0 < max_sequence_length < length:\n        length = max_sequence_length  # No generation bigger than model size\n    elif length < 0:\n        length = MAX_LENGTH  # avoid infinite loop\n\n\nSUMMARY:\n**Summary:**\n\n`adjust_length_to_model()` function adjusts the input `length` to ensure it's within the bounds of the maximum sequence length set by `max_sequence_length`. The function handles cases where `length` is negative or exceeds `max_sequence_length`.\n\n**Key Implementation Details:**\n\n* Returns adjusted `length` between 0 and `max_sequence_length`.\n* Handles edge cases:\n\t+ Negative `length`: sets to `MAX_LENGTH` avoiding infinite loop.\n\t+ Exceeds `max_sequence_length`: no generation larger than model size.\n\n**Notable Dependencies or Requirements:**\n\n* No external dependencies specified.\n* Uses a constant `MAX_LENGTH`."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:160-186

metadata:
{"docstring": "", "end_line": 186, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "sparse_model_config", "start_line": 160, "type": "function", "chroma:document": "CODE:\n\ndef sparse_model_config(model_config):\n    embedding_size = None\n    if hasattr(model_config, \"hidden_size\"):\n        embedding_size = model_config.hidden_size\n    elif hasattr(model_config, \"n_embed\"):\n        embedding_size = model_config.n_embed\n    elif hasattr(model_config, \"n_embd\"):\n        embedding_size = model_config.n_embd\n\n    num_head = None\n    if hasattr(model_config, \"num_attention_heads\"):\n        num_head = model_config.num_attention_heads\n    elif hasattr(model_config, \"n_head\"):\n        num_head = model_config.n_head\n\n    if embedding_size is None or num_head is None or num_head == 0:\n        raise ValueError(\"Check the model config\")\n\n    num_embedding_size_per_head = int(embedding_size / num_head)\n    if hasattr(model_config, \"n_layer\"):\n        num_layer = model_config.n_layer\n    elif hasattr(model_config, \"num_hidden_layers\"):\n        num_layer = model_config.num_hidden_layers\n    else:\n        raise ValueError(\"Number of hidden layers couldn't be determined from the model config\")\n\n\n\nSUMMARY:\n**Concise Summary**\n\nThe `sparse_model_config` function retrieves configuration settings for a sparse model, specifically `num_layers`, `num_heads`, and `num_embedding_size_per_head`. It extracts these values from the input `model_config` object, handling variable attribute names.\n\n**Key Details:**\n\n* Extracts:\n\t+ `embedding_size`: Determined by `hidden_size`, `n_embed`, or `n_embd`\n\t+ `num_head`: Determined by `num_attention_heads` or `n_head`\n* Checks for valid values and raises `ValueError` if any are invalid.\n* Calculates `num_embedding_size_per_head`\n\n**Notable Attributes/Functions:**\n\n* `model_config.hidden_size`, `model_config.n_embed`, `model_config.n_embd` (alternative names)\n* `model_config.num_attention_heads`, `model_config.n_head`\n* `model_config.n_layer`, `model_config.num_hidden_layers`\n\n**External Dependencies:**\n\nNone."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:189-215

metadata:
{"docstring": "", "end_line": 215, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "generate_past_key_values", "start_line": 189, "type": "function", "chroma:document": "CODE:\n\ndef generate_past_key_values(model, batch_size, seq_len):\n    num_block_layers, num_attention_heads, num_embedding_size_per_head = sparse_model_config(model.config)\n    if model.config.model_type == \"bloom\":\n        past_key_values = tuple(\n            (\n                torch.empty(int(num_attention_heads * batch_size), num_embedding_size_per_head, seq_len)\n                .to(model.dtype)\n                .to(model.device),\n                torch.empty(int(num_attention_heads * batch_size), seq_len, num_embedding_size_per_head)\n                .to(model.dtype)\n                .to(model.device),\n            )\n            for _ in range(num_block_layers)\n        )\n    else:\n        past_key_values = tuple(\n            (\n                torch.empty(batch_size, num_attention_heads, seq_len, num_embedding_size_per_head)\n                .to(model.dtype)\n                .to(model.device),\n                torch.empty(batch_size, num_attention_heads, seq_len, num_embedding_size_per_head)\n                .to(model.dtype)\n                .to(model.device),\n            )\n            for _ in range(num_block_layers)\n        )\n\n\nSUMMARY:\n**Summary: generate_past_key_values function**\n\n* **Purpose**: Generates key values and queries tensors to be used in Transformer model, specifically for multi-head attention mechanisms in different model architectures (\"bloom\" and general).\n\n* **Detailed implementation**:\n\t+ Uses `torch.empty` to create tensors.\n\t+ Sets tensor dimensions based on hyperparameters (`num_block_layers`, `batch_size`, `seq_len`, `num_attention_heads`, and `num_embedding_size_per_head`).\n\t+ Appends dimension `models.dtype` to ensure correct data type conversion.\n\t+ Moves tensors to device using `model.device`.\n\t+ Creates two separate sets of key values (self-attention queries) for each block layer, depending on the model architecture.\n\n* **Notable dependencies/requirements**:\n\t+ `torch`: The PyTorch library is required for creating and manipulating tensors.\n\t+ `sparse_model_config`: A function defining the configuration of sparse models.\n\t+ `model.config.model_type` with value \"bloom\" or otherwise, determining the Tensor shape allocation.\n\n* **Code quality**: Variable naming follows conventional Python naming conventions, but could be improved by including descriptive comments or docstrings to highlight individual algorithmic components."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:218-233

metadata:
{"docstring": "", "end_line": 233, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "prepare_jit_inputs", "start_line": 218, "type": "function", "chroma:document": "CODE:\n\ndef prepare_jit_inputs(inputs, model, tokenizer):\n    batch_size = len(inputs)\n    dummy_input = tokenizer.batch_encode_plus(inputs, return_tensors=\"pt\")\n    dummy_input = dummy_input.to(model.device)\n    if model.config.use_cache:\n        dummy_input[\"past_key_values\"] = generate_past_key_values(model, batch_size, 1)\n    dummy_input[\"attention_mask\"] = torch.cat(\n        [\n            torch.zeros(dummy_input[\"attention_mask\"].shape[0], 1)\n            .to(dummy_input[\"attention_mask\"].dtype)\n            .to(model.device),\n            dummy_input[\"attention_mask\"],\n        ],\n        -1,\n    )\n\n\nSUMMARY:\n**Concise Summary**\n\nThe `prepare_jit_inputs` function prepares inputs for a JIT (Just-In-Time) compilation process in a deep learning model. It:\n\n* Takes 3 inputs: `inputs`, `model`, and `tokenizer`.\n* Processes `inputs` into a `dummy_input` tokenized using the `tokenizer`, with attention mask created.\n* Appends padding to the beginning of the attention mask.\n* Returns the prepared `dummy_input`.\n\n**Key Details**\n\n* Tokenization using `tokenizer.batch_encode_plus`\n* Creation of dummy input tensor using `torch.cat`\n* Modification of attention mask using a special token (zero)\n\n**Dependencies/Requirements**\n\n* `inputs`, `model`, and `tokenizer` objects\n* `torch` library for tensor operations"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:236-280

metadata:
{"docstring": "", "end_line": 280, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "_ModelFallbackWrapper", "start_line": 236, "type": "class", "chroma:document": "CODE:\n\nclass _ModelFallbackWrapper(GenerationMixin):\n    __slots__ = (\"_optimized\", \"_default\")\n\n    def __init__(self, optimized, default):\n        self._optimized = optimized\n        self._default = default\n\n    def __call__(self, *args, **kwargs):\n        if kwargs[\"past_key_values\"] is None and self._default.config.use_cache:\n            kwargs[\"past_key_values\"] = generate_past_key_values(self._default, kwargs[\"input_ids\"].shape[0], 0)\n        kwargs.pop(\"position_ids\", None)\n        for k in list(kwargs.keys()):\n            if kwargs[k] is None or isinstance(kwargs[k], bool):\n                kwargs.pop(k)\n        outputs = self._optimized(**kwargs)\n        lm_logits = outputs[0]\n        past_key_values = outputs[1]\n        fixed_output = CausalLMOutputWithPast(\n            loss=None,\n            logits=lm_logits,\n            past_key_values=past_key_values,\n            hidden_states=None,\n            attentions=None,\n        )\n        return fixed_output\n\n    def __getattr__(self, item):\n        return getattr(self._default, item)\n\n    def prepare_inputs_for_generation(\n        self, input_ids, past_key_values=None, inputs_embeds=None, use_cache=None, **kwargs\n    ):\n        return self._default.prepare_inputs_for_generation(\n            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, **kwargs\n        )\n\n    def _reorder_cache(\n        self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor\n    ) -> Tuple[Tuple[torch.Tensor]]:\n        \"\"\"\n        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or\n        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct\n        beam_idx at every generation step.\n        \"\"\"\n\n\nSUMMARY:\n**Class Summary: `_ModelFallbackWrapper`**\n\nThe `_ModelFallbackWrapper` class is a wrapper that provides fallback behavior for generation models. Its primary purpose is to enable cache reuse and reordering of past key values during beam search.\n\n**Key Implementation Details:**\n\n*   The class takes two parameters in its constructor: `optimized` and `default`, which are instances of classes extending `GenerationMixin`.\n*   The `__call__` method handles the core generation logic, ensuring that cache is reused when necessary and reordering past key values for correct beam search.\n*   The `prepare_inputs_for_generation` method delegates to the `_default` object's implementation, allowing the wrapper to adapt its behavior.\n\n**Notable Dependencies and Requirements:**\n\n*   The class relies on a CausalLMOutputWithPast class that is not defined in this snippet.\n*   It also uses the `torch` library for tensor operations and potentially other dependencies.\n\n**Interfaces and Inheritance:**\n\nThe `_ModelFallbackWrapper` inherits from `GenerationMixin` with additional functionality. Notably, it wraps around an instance of a parent class (`optimized` or `default`) that implements specific methods (e.g., `__call__`, `prepare_inputs_for_generation`)."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py:283-442

metadata:
{"docstring": "", "end_line": 442, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation.py", "name": "main", "start_line": 283, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--model_type\",\n        default=None,\n        type=str,\n        required=True,\n        help=\"Model type selected in the list: \" + \", \".join(MODEL_CLASSES.keys()),\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        default=None,\n        type=str,\n        required=True,\n        help=\"Path to pre-trained model or shortcut name selected in the list: \" + \", \".join(MODEL_CLASSES.keys()),\n    )\n\n    parser.add_argument(\"--prompt\", type=str, default=\"\")\n    parser.add_argument(\"--length\", type=int, default=20)\n    parser.add_argument(\"--stop_token\", type=str, default=None, help=\"Token at which text generation is stopped\")\n\n    parser.add_argument(\n        \"--temperature\",\n        type=float,\n        default=1.0,\n        help=\"temperature of 1.0 has no effect, lower tend toward greedy sampling\",\n    )\n    parser.add_argument(\n        \"--repetition_penalty\", type=float, default=1.0, help=\"primarily useful for CTRL model; in that case, use 1.2\"\n    )\n    parser.add_argument(\"--k\", type=int, default=0)\n    parser.add_argument(\"--p\", type=float, default=0.9)\n\n    parser.add_argument(\"--prefix\", type=str, default=\"\", help=\"Text added prior to input.\")\n    parser.add_argument(\"--padding_text\", type=str, default=\"\", help=\"Deprecated, the use of `--prefix` is preferred.\")\n    parser.add_argument(\"--xlm_language\", type=str, default=\"\", help=\"Optional language when used with the XLM model.\")\n\n    parser.add_argument(\"--seed\", type=int, default=42, help=\"random seed for initialization\")\n    parser.add_argument(\n        \"--use_cpu\",\n        action=\"store_true\",\n        help=\"Whether or not to use cpu. If set to False, \" \"we will use gpu/npu or mps device if available\",\n    )\n    parser.add_argument(\"--num_return_sequences\", type=int, default=1, help=\"The number of samples to generate.\")\n    parser.add_argument(\n        \"--fp16\",\n        action=\"store_true\",\n        help=\"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit\",\n    )\n    parser.add_argument(\"--jit\", action=\"store_true\", help=\"Whether or not to use jit trace to accelerate inference\")\n    args = parser.parse_args()\n\n    # Initialize the distributed state.\n    distributed_state = PartialState(cpu=args.use_cpu)\n\n    logger.warning(f\"device: {distributed_state.device}, 16-bits inference: {args.fp16}\")\n\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Initialize the model and tokenizer\n    try:\n        args.model_type = args.model_type.lower()\n        model_class, tokenizer_class = MODEL_CLASSES[args.model_type]\n    except KeyError:\n        raise KeyError(\"the model {} you specified is not supported. You are welcome to add it and open a PR :)\")\n\n    tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path)\n    if tokenizer.pad_token is None:\n        tokenizer.pad_token = tokenizer.eos_token\n    model = model_class.from_pretrained(args.model_name_or_path)\n\n    # Set the model to the right device\n    model.to(distributed_state.device)\n\n    if args.fp16:\n        model.half()\n    max_seq_length = getattr(model.config, \"max_position_embeddings\", 0)\n    args.length = adjust_length_to_model(args.length, max_sequence_length=max_seq_length)\n    logger.info(args)\n\n    prompt_text = args.prompt if args.prompt else input(\"Model prompt >>> \")\n\n    # Different models need different input formatting and/or extra arguments\n    requires_preprocessing = args.model_type in PREPROCESSING_FUNCTIONS.keys()\n    if requires_preprocessing:\n        prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type)\n        preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text)\n\n        if model.__class__.__name__ in [\"TransfoXLLMHeadModel\"]:\n            tokenizer_kwargs = {\"add_space_before_punct_symbol\": True}\n        else:\n            tokenizer_kwargs = {}\n\n        encoded_prompt = tokenizer.encode(\n            preprocessed_prompt_text, add_special_tokens=False, return_tensors=\"pt\", **tokenizer_kwargs\n        )\n    else:\n        prefix = args.prefix if args.prefix else args.padding_text\n        encoded_prompt = tokenizer.encode(prefix + prompt_text, add_special_tokens=False, return_tensors=\"pt\")\n    encoded_prompt = encoded_prompt.to(distributed_state.device)\n\n    if encoded_prompt.size()[-1] == 0:\n        input_ids = None\n    else:\n        input_ids = encoded_prompt\n\n    if args.jit:\n        jit_input_texts = [\"enable jit\"]\n        jit_inputs = prepare_jit_inputs(jit_input_texts, model, tokenizer)\n        torch._C._jit_set_texpr_fuser_enabled(False)\n        model.config.return_dict = False\n        if hasattr(model, \"forward\"):\n            sig = inspect.signature(model.forward)\n        else:\n            sig = inspect.signature(model.__call__)\n        jit_inputs = tuple(jit_inputs[key] for key in sig.parameters if jit_inputs.get(key, None) is not None)\n        traced_model = torch.jit.trace(model, jit_inputs, strict=False)\n        traced_model = torch.jit.freeze(traced_model.eval())\n        traced_model(*jit_inputs)\n        traced_model(*jit_inputs)\n\n        model = _ModelFallbackWrapper(traced_model, model)\n\n    output_sequences = model.generate(\n        input_ids=input_ids,\n        max_length=args.length + len(encoded_prompt[0]),\n        temperature=args.temperature,\n        top_k=args.k,\n        top_p=args.p,\n        repetition_penalty=args.repetition_penalty,\n        do_sample=True,\n        num_return_sequences=args.num_return_sequences,\n    )\n\n    # Remove the batch dimension when returning multiple sequences\n    if len(output_sequences.shape) > 2:\n        output_sequences.squeeze_()\n\n    generated_sequences = []\n\n    for generated_sequence_idx, generated_sequence in enumerate(output_sequences):\n        print(f\"=== GENERATED SEQUENCE {generated_sequence_idx + 1} ===\")\n        generated_sequence = generated_sequence.tolist()\n\n        # Decode text\n        text = tokenizer.decode(generated_sequence, clean_up_tokenization_spaces=True)\n\n        # Remove all text after the stop token\n        text = text[: text.find(args.stop_token) if args.stop_token else None]\n\n        # Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing\n        total_sequence = (\n            prompt_text + text[len(tokenizer.decode(encoded_prompt[0], clean_up_tokenization_spaces=True)) :]\n        )\n\n        generated_sequences.append(total_sequence)\n        print(total_sequence)\n\n\n\nSUMMARY:\n**Function Summary:**\n\nThis function generates sequences of text based on input prompts. It uses the Hugging Face Transformers library and is designed to be highly customizable.\n\n**Main Purpose and Functionality:**\n\nThe primary purpose of this function is to generate sequential text based on user input. The function takes in multiple command-line arguments, including:\n\n* `--model_type`: The type of model to use (e.g., CTRL, T5)\n* `--model_name_or_path`: The path to a pre-trained model or its name\n* `--prompt`: The user prompt for the text generation\n*`--length`, `--stop_token`, `--temperature`, `--repetition_penalty`, `--k`, and `--p`: Various parameters controlling the text generation process\n\n**Key Implementation Details:**\n\nThe function uses Model Classes from Hugging Face Transformers library. Key implementation details include:\n\n* Loading pre-trained models using `model_class.from_pretrained(args.model_name_or_path)`\n* Converting user input to input IDs using the tokenizer\n* Using the model's `generate()` method to produce text sequences based on the input IDs and provided parameters\n\n**Notable Dependencies or Requirements:**\n\nThis function relies on the Hugging Face Transformers library for model loading, tokenization, and text generation.\n\nExample external dependencies:\n```\nimport torch\nfrom transformers import Model, AutoTokenizer\n```\n\nThis summary focuses on the core functionality of the code, providing a concise overview of its main purpose, key implementation details, and notable dependencies."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation_contrastive_search.py:38-131

metadata:
{"docstring": "", "end_line": 131, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/text-generation/run_generation_contrastive_search.py", "name": "main", "start_line": 38, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\n        \"--model_name_or_path\",\n        default=None,\n        type=str,\n        required=True,\n    )\n    parser.add_argument(\"--prompt\", type=str, default=\"\")\n    parser.add_argument(\"--length\", type=int, default=20)\n    parser.add_argument(\"--stop_token\", type=str, default=None, help=\"Token at which text generation is stopped\")\n    parser.add_argument(\n        \"--temperature\",\n        type=float,\n        default=1.0,\n        help=\"temperature of 1.0 has no effect, lower tend toward greedy sampling\",\n    )\n    parser.add_argument(\n        \"--repetition_penalty\", type=float, default=1.0, help=\"primarily useful for CTRL model; in that case, use 1.2\"\n    )\n    parser.add_argument(\"--k\", type=int, default=0)\n    parser.add_argument(\"--penalty_alpha\", type=float, default=0.0)\n    parser.add_argument(\"--p\", type=float, default=0.9)\n\n    parser.add_argument(\"--prefix\", type=str, default=\"\", help=\"Text added prior to input.\")\n    parser.add_argument(\"--padding_text\", type=str, default=\"\", help=\"Deprecated, the use of `--prefix` is preferred.\")\n    parser.add_argument(\"--xlm_language\", type=str, default=\"\", help=\"Optional language when used with the XLM model.\")\n\n    parser.add_argument(\"--seed\", type=int, default=42, help=\"random seed for initialization\")\n    parser.add_argument(\n        \"--use_cpu\",\n        action=\"store_true\",\n        help=\"Whether or not to use cpu. If set to False, \" \"we will use gpu/npu or mps device if available\",\n    )\n    parser.add_argument(\n        \"--fp16\",\n        action=\"store_true\",\n        help=\"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit\",\n    )\n    args = parser.parse_args()\n\n    # Initialize the distributed state.\n    distributed_state = PartialState(cpu=args.use_cpu)\n\n    logger.warning(f\"device: {distributed_state.device}, 16-bits inference: {args.fp16}\")\n\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Initialize the model and tokenizer\n    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)\n    model = AutoModelForCausalLM.from_pretrained(args.model_name_or_path)\n\n    # tokenizer = GPT2Tokenizer.from_pretrained(args.model_name_or_path)\n    # model = OPTForCausalLM.from_pretrained(args.model_name_or_path)\n    # Set the model to the right device\n    model.to(distributed_state.device)\n\n    if args.fp16:\n        model.half()\n\n    logger.info(args)\n    prompt_text = args.prompt if args.prompt else input(\"Model prompt >>> \")\n\n    inputs = tokenizer(prompt_text, return_tensors=\"pt\", add_special_tokens=False)\n    inputs = {key: value.to(distributed_state.device) for key, value in inputs.items()}\n\n    output_sequences = model.generate(\n        **inputs,\n        max_length=args.length + len(inputs[\"input_ids\"][0]),\n        penalty_alpha=args.penalty_alpha,\n        top_k=args.k,\n    )\n\n    generated_sequences = []\n    for generated_sequence_idx, generated_sequence in enumerate(output_sequences):\n        print(f\"=== GENERATED SEQUENCE {generated_sequence_idx + 1} ===\")\n        generated_sequence = generated_sequence.tolist()\n\n        # Decode text\n        text = tokenizer.decode(generated_sequence, clean_up_tokenization_spaces=True, add_special_tokens=False)\n\n        # Remove all text after the stop token\n        text = text[: text.find(args.stop_token) if args.stop_token else None]\n\n        # Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing\n        total_sequence = (\n            prompt_text + text[len(tokenizer.decode(inputs[\"input_ids\"][0], clean_up_tokenization_spaces=True)) :]\n        )\n\n        generated_sequences.append(total_sequence)\n        print(total_sequence)\n\n\n\nSUMMARY:\n**Summary:**\n\nThis is the main function for generating text sequences using a pre-trained language model.\n\n**Primary Purpose:** Generate text sequences given a prompt and model configuration.\n\n**Key Implementation Details:**\n\n* Initializes an `ArgumentParser` to parse command-line arguments for:\n\t+ Model name or path\n\t+ Prompt text\n\t+ Sequence length\n\t+ Stop token\n\t+ Temperature and repetition penalty parameters\n\t+ Additional configuration options (prefix, padding text, language code)\n* Sets the random seed for initialization if specified\n* Initializes the distributed state and model tokenizer and model using Hugging Face Transformers library\n* Generates text sequences using the `generate` method of the model with the given inputs\n\n**Notable Dependencies:**\n\n* Hugging Face Transformers library (specifically, `argparse`, `AutoTokenizer`, `AutoModelForCausalLM`)\n* External dependencies:\n\t+ NVIDIA Apex for 16-bit precision (if --fp16 is used)\n\n**Key Method Signatures and Interfaces:**\n\n* `generate(model=self.model, ...)` from Hugging Face Transformers library\n* `set_seed(seed)` from a custom function (not shown in this code snippet)\n* `decode(generated_sequence, ...)`, `add_special_tokens`, `clean_up_tokenization_spaces` methods from Hugging Face Transformers library"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag_no_trainer.py:66-225

metadata:
{"docstring": "", "end_line": 225, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag_no_trainer.py", "name": "parse_args", "start_line": 66, "type": "function", "chroma:document": "CODE:\n\ndef parse_args():\n    parser = argparse.ArgumentParser(description=\"Finetune a transformers model on a multiple choice task\")\n    parser.add_argument(\n        \"--dataset_name\",\n        type=str,\n        default=None,\n        help=\"The name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--dataset_config_name\",\n        type=str,\n        default=None,\n        help=\"The configuration name of the dataset to use (via the datasets library).\",\n    )\n    parser.add_argument(\n        \"--train_file\", type=str, default=None, help=\"A csv or a json file containing the training data.\"\n    )\n    parser.add_argument(\n        \"--validation_file\", type=str, default=None, help=\"A csv or a json file containing the validation data.\"\n    )\n    parser.add_argument(\n        \"--max_seq_length\",\n        type=int,\n        default=128,\n        help=(\n            \"The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,\"\n            \" sequences shorter will be padded if `--pad_to_max_length` is passed.\"\n        ),\n    )\n    parser.add_argument(\n        \"--pad_to_max_length\",\n        action=\"store_true\",\n        help=\"If passed, pad all samples to `max_length`. Otherwise, dynamic padding is used.\",\n    )\n    parser.add_argument(\n        \"--model_name_or_path\",\n        type=str,\n        help=\"Path to pretrained model or model identifier from huggingface.co/models.\",\n        required=False,\n    )\n    parser.add_argument(\n        \"--config_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained config name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--tokenizer_name\",\n        type=str,\n        default=None,\n        help=\"Pretrained tokenizer name or path if not the same as model_name\",\n    )\n    parser.add_argument(\n        \"--use_slow_tokenizer\",\n        action=\"store_true\",\n        help=\"If passed, will use a slow tokenizer (not backed by the \ud83e\udd17 Tokenizers library).\",\n    )\n    parser.add_argument(\n        \"--per_device_train_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the training dataloader.\",\n    )\n    parser.add_argument(\n        \"--per_device_eval_batch_size\",\n        type=int,\n        default=8,\n        help=\"Batch size (per device) for the evaluation dataloader.\",\n    )\n    parser.add_argument(\n        \"--learning_rate\",\n        type=float,\n        default=5e-5,\n        help=\"Initial learning rate (after the potential warmup period) to use.\",\n    )\n    parser.add_argument(\"--weight_decay\", type=float, default=0.0, help=\"Weight decay to use.\")\n    parser.add_argument(\"--num_train_epochs\", type=int, default=3, help=\"Total number of training epochs to perform.\")\n    parser.add_argument(\n        \"--max_train_steps\",\n        type=int,\n        default=None,\n        help=\"Total number of training steps to perform. If provided, overrides num_train_epochs.\",\n    )\n    parser.add_argument(\n        \"--gradient_accumulation_steps\",\n        type=int,\n        default=1,\n        help=\"Number of updates steps to accumulate before performing a backward/update pass.\",\n    )\n    parser.add_argument(\n        \"--lr_scheduler_type\",\n        type=SchedulerType,\n        default=\"linear\",\n        help=\"The scheduler type to use.\",\n        choices=[\"linear\", \"cosine\", \"cosine_with_restarts\", \"polynomial\", \"constant\", \"constant_with_warmup\"],\n    )\n    parser.add_argument(\n        \"--num_warmup_steps\", type=int, default=0, help=\"Number of steps for the warmup in the lr scheduler.\"\n    )\n    parser.add_argument(\"--output_dir\", type=str, default=None, help=\"Where to store the final model.\")\n    parser.add_argument(\"--seed\", type=int, default=None, help=\"A seed for reproducible training.\")\n    parser.add_argument(\n        \"--model_type\",\n        type=str,\n        default=None,\n        help=\"Model type to use if training from scratch.\",\n        choices=MODEL_TYPES,\n    )\n    parser.add_argument(\n        \"--debug\",\n        action=\"store_true\",\n        help=\"Activate debug mode and run training only with a subset of data.\",\n    )\n    parser.add_argument(\"--push_to_hub\", action=\"store_true\", help=\"Whether or not to push the model to the Hub.\")\n    parser.add_argument(\n        \"--hub_model_id\", type=str, help=\"The name of the repository to keep in sync with the local `output_dir`.\"\n    )\n    parser.add_argument(\"--hub_token\", type=str, help=\"The token to use to push to the Model Hub.\")\n    parser.add_argument(\n        \"--trust_remote_code\",\n        action=\"store_true\",\n        help=(\n            \"Whether to trust the execution of code from datasets/models defined on the Hub.\"\n            \" This option should only be set to `True` for repositories you trust and in which you have read the\"\n            \" code, as it will execute code present on the Hub on your local machine.\"\n        ),\n    )\n    parser.add_argument(\n        \"--checkpointing_steps\",\n        type=str,\n        default=None,\n        help=\"Whether the various states should be saved at the end of every n steps, or 'epoch' for each epoch.\",\n    )\n    parser.add_argument(\n        \"--resume_from_checkpoint\",\n        type=str,\n        default=None,\n        help=\"If the training should continue from a checkpoint folder.\",\n    )\n    parser.add_argument(\n        \"--with_tracking\",\n        action=\"store_true\",\n        help=\"Whether to enable experiment trackers for logging.\",\n    )\n    parser.add_argument(\n        \"--report_to\",\n        type=str,\n        default=\"all\",\n        help=(\n            'The integration to report the results and logs to. Supported platforms are `\"tensorboard\"`,'\n            ' `\"wandb\"`, `\"comet_ml\"` and `\"clearml\"`. Use `\"all\"` (default) to report to all integrations. '\n            \"Only applicable when `--with_tracking` is passed.\"\n        ),\n    )\n    args = parser.parse_args()\n\n    if args.push_to_hub:\n        assert args.output_dir is not None, \"Need an `output_dir` to create a repo when `--push_to_hub` is passed.\"\n\n\n\nSUMMARY:\n**summary**\n\nThis function creates an argument parser for fine-tuning a transformer model on a multiple choice task. It supports various command-line arguments, including:\n\n* Dataset and configuration names\n* Training data file paths\n* Model parameters (e.g., learning rate, batch sizes)\n* Output directory and other experiment settings\n\n**Key details**\n\n* The `argparse.ArgumentParser` class is used to define the argument parser.\n* 33 arguments are defined, with varying data types and default values.\n\n**Notable dependencies**\n\n* `datasets` library for dataset configuration\n* `huggingface.co/models` for model selection\n* `Tokenizers` library (optional)"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag_no_trainer.py:285-703

metadata:
{"docstring": "", "end_line": 703, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag_no_trainer.py", "name": "main", "start_line": 285, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    args = parse_args()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_swag_no_trainer\", args)\n\n    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.\n    # If we're using tracking, we also need to initialize it here and it will by default pick up all supported trackers\n    # in the environment\n    accelerator_log_kwargs = {}\n\n    if args.with_tracking:\n        accelerator_log_kwargs[\"log_with\"] = args.report_to\n        accelerator_log_kwargs[\"project_dir\"] = args.output_dir\n\n    accelerator = Accelerator(gradient_accumulation_steps=args.gradient_accumulation_steps, **accelerator_log_kwargs)\n\n    # Make one log on every process with the configuration for debugging.\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        level=logging.INFO,\n    )\n    logger.info(accelerator.state, main_process_only=False)\n    if accelerator.is_local_main_process:\n        datasets.utils.logging.set_verbosity_warning()\n        transformers.utils.logging.set_verbosity_info()\n    else:\n        datasets.utils.logging.set_verbosity_error()\n        transformers.utils.logging.set_verbosity_error()\n\n    # If passed along, set the training seed now.\n    if args.seed is not None:\n        set_seed(args.seed)\n\n    # Handle the repository creation\n    if accelerator.is_main_process:\n        if args.push_to_hub:\n            # Retrieve of infer repo_name\n            repo_name = args.hub_model_id\n            if repo_name is None:\n                repo_name = Path(args.output_dir).absolute().name\n            # Create repo and retrieve repo_id\n            api = HfApi()\n            repo_id = api.create_repo(repo_name, exist_ok=True, token=args.hub_token).repo_id\n\n            with open(os.path.join(args.output_dir, \".gitignore\"), \"w+\") as gitignore:\n                if \"step_*\" not in gitignore:\n                    gitignore.write(\"step_*\\n\")\n                if \"epoch_*\" not in gitignore:\n                    gitignore.write(\"epoch_*\\n\")\n        elif args.output_dir is not None:\n            os.makedirs(args.output_dir, exist_ok=True)\n    accelerator.wait_for_everyone()\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n    #\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        raw_datasets = load_dataset(\n            args.dataset_name, args.dataset_config_name, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        data_files = {}\n        if args.train_file is not None:\n            data_files[\"train\"] = args.train_file\n            extension = args.train_file.split(\".\")[-1]\n        if args.validation_file is not None:\n            data_files[\"validation\"] = args.validation_file\n            extension = args.validation_file.split(\".\")[-1]\n        raw_datasets = load_dataset(extension, data_files=data_files)\n    # Trim a number of training examples\n    if args.debug:\n        for split in raw_datasets.keys():\n            raw_datasets[split] = raw_datasets[split].select(range(100))\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    if raw_datasets[\"train\"] is not None:\n        column_names = raw_datasets[\"train\"].column_names\n    else:\n        column_names = raw_datasets[\"validation\"].column_names\n\n    # When using your own dataset or a different dataset from swag, you will probably need to change this.\n    ending_names = [f\"ending{i}\" for i in range(4)]\n    context_name = \"sent1\"\n    question_header_name = \"sent2\"\n    label_column_name = \"label\" if \"label\" in column_names else \"labels\"\n\n    # Load pretrained model and tokenizer\n    #\n    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    if args.config_name:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    elif args.model_name_or_path:\n        config = AutoConfig.from_pretrained(args.model_name_or_path, trust_remote_code=args.trust_remote_code)\n    else:\n        config = CONFIG_MAPPING[args.model_type]()\n        logger.warning(\"You are instantiating a new config instance from scratch.\")\n\n    if args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.tokenizer_name, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    elif args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            args.model_name_or_path, use_fast=not args.use_slow_tokenizer, trust_remote_code=args.trust_remote_code\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    if args.model_name_or_path:\n        model = AutoModelForMultipleChoice.from_pretrained(\n            args.model_name_or_path,\n            from_tf=bool(\".ckpt\" in args.model_name_or_path),\n            config=config,\n            trust_remote_code=args.trust_remote_code,\n        )\n    else:\n        logger.info(\"Training new model from scratch\")\n        model = AutoModelForMultipleChoice.from_config(config, trust_remote_code=args.trust_remote_code)\n\n    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch\n    # on a small vocab and want a smaller embedding size, remove this test.\n    embedding_size = model.get_input_embeddings().weight.shape[0]\n    if len(tokenizer) > embedding_size:\n        model.resize_token_embeddings(len(tokenizer))\n\n    # Preprocessing the datasets.\n    # First we tokenize all the texts.\n    padding = \"max_length\" if args.pad_to_max_length else False\n\n    def preprocess_function(examples):\n        first_sentences = [[context] * 4 for context in examples[context_name]]\n        question_headers = examples[question_header_name]\n        second_sentences = [\n            [f\"{header} {examples[end][i]}\" for end in ending_names] for i, header in enumerate(question_headers)\n        ]\n        labels = examples[label_column_name]\n\n        # Flatten out\n        first_sentences = list(chain(*first_sentences))\n        second_sentences = list(chain(*second_sentences))\n\n        # Tokenize\n        tokenized_examples = tokenizer(\n            first_sentences,\n            second_sentences,\n            max_length=args.max_seq_length,\n            padding=padding,\n            truncation=True,\n        )\n        # Un-flatten\n        tokenized_inputs = {k: [v[i : i + 4] for i in range(0, len(v), 4)] for k, v in tokenized_examples.items()}\n        tokenized_inputs[\"labels\"] = labels\n        return tokenized_inputs\n\n    with accelerator.main_process_first():\n        processed_datasets = raw_datasets.map(\n            preprocess_function, batched=True, remove_columns=raw_datasets[\"train\"].column_names\n        )\n\n    train_dataset = processed_datasets[\"train\"]\n    eval_dataset = processed_datasets[\"validation\"]\n\n    # Log a few random samples from the training set:\n    for index in random.sample(range(len(train_dataset)), 3):\n        logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n\n    # DataLoaders creation:\n    if args.pad_to_max_length:\n        # If padding was already done ot max length, we use the default data collator that will just convert everything\n        # to tensors.\n        data_collator = default_data_collator\n    else:\n        # Otherwise, `DataCollatorWithPadding` will apply dynamic padding for us (by padding to the maximum length of\n        # the samples passed). When using mixed precision, we add `pad_to_multiple_of=8` to pad all tensors to multiple\n        # of 8s, which will enable the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).\n        # For fp8, we pad to multiple of 16.\n        if accelerator.mixed_precision == \"fp8\":\n            pad_to_multiple_of = 16\n        elif accelerator.mixed_precision != \"no\":\n            pad_to_multiple_of = 8\n        else:\n            pad_to_multiple_of = None\n        data_collator = DataCollatorForMultipleChoice(tokenizer, pad_to_multiple_of=pad_to_multiple_of)\n\n    train_dataloader = DataLoader(\n        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size\n    )\n    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)\n\n    # Optimizer\n    # Split weights in two groups, one with weight decay and the other not.\n    no_decay = [\"bias\", \"LayerNorm.weight\"]\n    optimizer_grouped_parameters = [\n        {\n            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n            \"weight_decay\": args.weight_decay,\n        },\n        {\n            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n            \"weight_decay\": 0.0,\n        },\n    ]\n    optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=args.learning_rate)\n\n    # Use the device given by the `accelerator` object.\n    device = accelerator.device\n    model.to(device)\n\n    # Scheduler and math around the number of training steps.\n    overrode_max_train_steps = False\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if args.max_train_steps is None:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n        overrode_max_train_steps = True\n\n    lr_scheduler = get_scheduler(\n        name=args.lr_scheduler_type,\n        optimizer=optimizer,\n        num_warmup_steps=args.num_warmup_steps * accelerator.num_processes,\n        num_training_steps=args.max_train_steps\n        if overrode_max_train_steps\n        else args.max_train_steps * accelerator.num_processes,\n    )\n\n    # Prepare everything with our `accelerator`.\n    model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(\n        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler\n    )\n\n    # We need to recalculate our total training steps as the size of the training dataloader may have changed.\n    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)\n    if overrode_max_train_steps:\n        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch\n    # Afterwards we recalculate our number of training epochs\n    args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)\n\n    # Figure out how many steps we should save the Accelerator states\n    checkpointing_steps = args.checkpointing_steps\n    if checkpointing_steps is not None and checkpointing_steps.isdigit():\n        checkpointing_steps = int(checkpointing_steps)\n\n    # We need to initialize the trackers we use, and also store our configuration.\n    # The trackers initializes automatically on the main process.\n    if args.with_tracking:\n        experiment_config = vars(args)\n        # TensorBoard cannot log Enums, need the raw value\n        experiment_config[\"lr_scheduler_type\"] = experiment_config[\"lr_scheduler_type\"].value\n        accelerator.init_trackers(\"swag_no_trainer\", experiment_config)\n\n    # Metrics\n    metric = evaluate.load(\"accuracy\")\n\n    # Train!\n    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps\n\n    logger.info(\"***** Running training *****\")\n    logger.info(f\"  Num examples = {len(train_dataset)}\")\n    logger.info(f\"  Num Epochs = {args.num_train_epochs}\")\n    logger.info(f\"  Instantaneous batch size per device = {args.per_device_train_batch_size}\")\n    logger.info(f\"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}\")\n    logger.info(f\"  Gradient Accumulation steps = {args.gradient_accumulation_steps}\")\n    logger.info(f\"  Total optimization steps = {args.max_train_steps}\")\n    # Only show the progress bar once on each machine.\n    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)\n    completed_steps = 0\n    starting_epoch = 0\n\n    # Potentially load in the weights and states from a previous save\n    if args.resume_from_checkpoint:\n        if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != \"\":\n            checkpoint_path = args.resume_from_checkpoint\n            path = os.path.basename(args.resume_from_checkpoint)\n        else:\n            # Get the most recent checkpoint\n            dirs = [f.name for f in os.scandir(os.getcwd()) if f.is_dir()]\n            dirs.sort(key=os.path.getctime)\n            path = dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last\n            checkpoint_path = path\n            path = os.path.basename(checkpoint_path)\n\n        accelerator.print(f\"Resumed from checkpoint: {checkpoint_path}\")\n        accelerator.load_state(checkpoint_path)\n        # Extract `epoch_{i}` or `step_{i}`\n        training_difference = os.path.splitext(path)[0]\n\n        if \"epoch\" in training_difference:\n            starting_epoch = int(training_difference.replace(\"epoch_\", \"\")) + 1\n            resume_step = None\n            completed_steps = starting_epoch * num_update_steps_per_epoch\n        else:\n            # need to multiply `gradient_accumulation_steps` to reflect real steps\n            resume_step = int(training_difference.replace(\"step_\", \"\")) * args.gradient_accumulation_steps\n            starting_epoch = resume_step // len(train_dataloader)\n            completed_steps = resume_step // args.gradient_accumulation_steps\n            resume_step -= starting_epoch * len(train_dataloader)\n\n    # update the progress_bar if load from checkpoint\n    progress_bar.update(completed_steps)\n\n    for epoch in range(starting_epoch, args.num_train_epochs):\n        model.train()\n        if args.with_tracking:\n            total_loss = 0\n        if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:\n            # We skip the first `n` batches in the dataloader when resuming from a checkpoint\n            active_dataloader = accelerator.skip_first_batches(train_dataloader, resume_step)\n        else:\n            active_dataloader = train_dataloader\n        for step, batch in enumerate(active_dataloader):\n            with accelerator.accumulate(model):\n                outputs = model(**batch)\n                loss = outputs.loss\n                # We keep track of the loss at each epoch\n                if args.with_tracking:\n                    total_loss += loss.detach().float()\n                accelerator.backward(loss)\n                optimizer.step()\n                lr_scheduler.step()\n                optimizer.zero_grad()\n\n            # Checks if the accelerator has performed an optimization step behind the scenes\n            if accelerator.sync_gradients:\n                progress_bar.update(1)\n                completed_steps += 1\n\n            if isinstance(checkpointing_steps, int):\n                if completed_steps % checkpointing_steps == 0 and accelerator.sync_gradients:\n                    output_dir = f\"step_{completed_steps}\"\n                    if args.output_dir is not None:\n                        output_dir = os.path.join(args.output_dir, output_dir)\n                    accelerator.save_state(output_dir)\n\n            if completed_steps >= args.max_train_steps:\n                break\n\n        model.eval()\n        for step, batch in enumerate(eval_dataloader):\n            with torch.no_grad():\n                outputs = model(**batch)\n            predictions = outputs.logits.argmax(dim=-1)\n            predictions, references = accelerator.gather_for_metrics((predictions, batch[\"labels\"]))\n            metric.add_batch(\n                predictions=predictions,\n                references=references,\n            )\n\n        eval_metric = metric.compute()\n        accelerator.print(f\"epoch {epoch}: {eval_metric}\")\n\n        if args.with_tracking:\n            accelerator.log(\n                {\n                    \"accuracy\": eval_metric,\n                    \"train_loss\": total_loss.item() / len(train_dataloader),\n                    \"epoch\": epoch,\n                    \"step\": completed_steps,\n                },\n                step=completed_steps,\n            )\n\n        if args.push_to_hub and epoch < args.num_train_epochs - 1:\n            accelerator.wait_for_everyone()\n            unwrapped_model = accelerator.unwrap_model(model)\n            unwrapped_model.save_pretrained(\n                args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n            )\n            if accelerator.is_main_process:\n                tokenizer.save_pretrained(args.output_dir)\n                api.upload_folder(\n                    commit_message=f\"Training in progress epoch {epoch}\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n\n        if args.checkpointing_steps == \"epoch\":\n            output_dir = f\"epoch_{epoch}\"\n            if args.output_dir is not None:\n                output_dir = os.path.join(args.output_dir, output_dir)\n            accelerator.save_state(output_dir)\n\n    if args.with_tracking:\n        accelerator.end_training()\n\n    if args.output_dir is not None:\n        accelerator.wait_for_everyone()\n        unwrapped_model = accelerator.unwrap_model(model)\n        unwrapped_model.save_pretrained(\n            args.output_dir, is_main_process=accelerator.is_main_process, save_function=accelerator.save\n        )\n        if accelerator.is_main_process:\n            tokenizer.save_pretrained(args.output_dir)\n            if args.push_to_hub:\n                api.upload_folder(\n                    commit_message=\"End of training\",\n                    folder_path=args.output_dir,\n                    repo_id=repo_id,\n                    repo_type=\"model\",\n                    token=args.hub_token,\n                )\n            all_results = {f\"eval_{k}\": v for k, v in eval_metric.items()}\n            with open(os.path.join(args.output_dir, \"all_results.json\"), \"w\") as f:\n\n\nSUMMARY:\nThe code is a PyTorch implementation of a neural network training pipeline using the Hugging Face Transformers library and the Weights & Biases tracking system. The pipeline trains a model on a dataset using a specified hyperparameter search space.\n\nHere's a high-level overview of the pipeline:\n\n1. Data loading: The code loads the dataset into memory.\n2. Model initialization: A pre-trained transformer model is initialized with a specific configuration.\n3. Hyperparameter search: The code performs a grid search over a defined hyperparameter space to find the best combination that results in a minimum loss.\n4. Training loop: The model is trained on the dataset in a loop, with optimization and logging steps at each epoch.\n5. Evaluation: After training finishes, the model is evaluated on a validation set and the accuracy is computed.\n6. Outputting results: The final results are saved to disk, including the trained model checkpoints, tokenizer, and evaluation metrics.\n\nKey components:\n\n* `PyTorch` for neural network building and optimization\n* `Hugging Face Transformers` for pre-trained models and tokenizers\n* `Weights & Biases` for tracking pipeline runs and hyperparameter search\n* `API` for uploading results to the Hugging Face model hub\n\nThis code is likely used in a research or development environment where the goal is to train high-performance language models for tasks such as natural language processing, computer vision, etc."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag.py:224-490
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 490, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag.py", "name": "main", "start_line": 224, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_swag\", model_args, data_args)\n\n    # Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    datasets.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # Detecting last checkpoint.\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # Set seed before initializing model.\n    set_seed(training_args.seed)\n\n    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n\n    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called\n    # 'text' is found. You can easily tweak this behavior (see below).\n\n    # In distributed training, the load_dataset function guarantee that only one local process can concurrently\n    # download the dataset.\n    if data_args.train_file is not None or data_args.validation_file is not None:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        raw_datasets = load_dataset(\n            extension,\n            data_files=data_files,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    else:\n        # Downloading and loading the swag dataset from the hub.\n        raw_datasets = load_dataset(\n            \"swag\",\n            \"regular\",\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # Load pretrained model and tokenizer\n\n    # Distributed training:\n    # The .from_pretrained methods guarantee that only one local process can concurrently\n    # download model & vocab.\n    config = AutoConfig.from_pretrained(\n        model_args.config_name if model_args.config_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    tokenizer = AutoTokenizer.from_pretrained(\n        model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        use_fast=model_args.use_fast_tokenizer,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    model = AutoModelForMultipleChoice.from_pretrained(\n        model_args.model_name_or_path,\n        from_tf=bool(\".ckpt\" in model_args.model_name_or_path),\n        config=config,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    # When using your own dataset or a different dataset from swag, you will probably need to change this.\n    ending_names = [f\"ending{i}\" for i in range(4)]\n    context_name = \"sent1\"\n    question_header_name = \"sent2\"\n\n    if data_args.max_seq_length is None:\n        max_seq_length = tokenizer.model_max_length\n        if max_seq_length > 1024:\n            logger.warning(\n                \"The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value\"\n                \" of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can\"\n                \" override this default with `--block_size xxx`.\"\n            )\n            max_seq_length = 1024\n    else:\n        if data_args.max_seq_length > tokenizer.model_max_length:\n            logger.warning(\n                f\"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the \"\n                f\"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}.\"\n            )\n        max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)\n\n    # Preprocessing the datasets.\n    def preprocess_function(examples):\n        first_sentences = [[context] * 4 for context in examples[context_name]]\n        question_headers = examples[question_header_name]\n        second_sentences = [\n            [f\"{header} {examples[end][i]}\" for end in ending_names] for i, header in enumerate(question_headers)\n        ]\n\n        # Flatten out\n        first_sentences = list(chain(*first_sentences))\n        second_sentences = list(chain(*second_sentences))\n\n        # Tokenize\n        tokenized_examples = tokenizer(\n            first_sentences,\n            second_sentences,\n            truncation=True,\n            max_length=max_seq_length,\n            padding=\"max_length\" if data_args.pad_to_max_length else False,\n        )\n        # Un-flatten\n        return {k: [v[i : i + 4] for i in range(0, len(v), 4)] for k, v in tokenized_examples.items()}\n\n    if training_args.do_train:\n        if \"train\" not in raw_datasets:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = raw_datasets[\"train\"]\n        if data_args.max_train_samples is not None:\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n        with training_args.main_process_first(desc=\"train dataset map pre-processing\"):\n            train_dataset = train_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                load_from_cache_file=not data_args.overwrite_cache,\n            )\n\n    if training_args.do_eval:\n        if \"validation\" not in raw_datasets:\n            raise ValueError(\"--do_eval requires a validation dataset\")\n        eval_dataset = raw_datasets[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n        with training_args.main_process_first(desc=\"validation dataset map pre-processing\"):\n            eval_dataset = eval_dataset.map(\n                preprocess_function,\n                batched=True,\n                num_proc=data_args.preprocessing_num_workers,\n                load_from_cache_file=not data_args.overwrite_cache,\n            )\n\n    # Data collator\n    data_collator = (\n        default_data_collator\n        if data_args.pad_to_max_length\n        else DataCollatorForMultipleChoice(tokenizer=tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None)\n    )\n\n    # Metric\n    def compute_metrics(eval_predictions):\n        predictions, label_ids = eval_predictions\n        preds = np.argmax(predictions, axis=1)\n        return {\"accuracy\": (preds == label_ids).astype(np.float32).mean().item()}\n\n    # Initialize our Trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        processing_class=tokenizer,\n        data_collator=data_collator,\n        compute_metrics=compute_metrics,\n    )\n\n    # Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()  # Saves the tokenizer too for easy upload\n        metrics = train_result.metrics\n\n        max_train_samples = (\n            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)\n        )\n        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n\n        trainer.log_metrics(\"train\", metrics)\n        trainer.save_metrics(\"train\", metrics)\n        trainer.save_state()\n\n    # Evaluation\n    if training_args.do_eval:\n        logger.info(\"*** Evaluate ***\")\n\n        metrics = trainer.evaluate()\n        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)\n        metrics[\"eval_samples\"] = min(max_eval_samples, len(eval_dataset))\n\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    kwargs = {\n        \"finetuned_from\": model_args.model_name_or_path,\n        \"tasks\": \"multiple-choice\",\n        \"dataset_tags\": \"swag\",\n        \"dataset_args\": \"regular\",\n        \"dataset\": \"SWAG\",\n        \"language\": \"en\",\n    }\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nHere is a concise summary:\n\nThis code sets up a Hugging Face Trainer for fine-tuning an English question-answering model on the SWAG (Stanford Question Answering Game) dataset. It defines multiple datasets, preprocesses them using the `tokenizer` and `pre ProcessFunction`, trains the model with a suitable datollarator and computing metric, saves trained weights to Hugging Face Model Hub after completion."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
id: /home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag.py:493-495
encoding: FLOAT32
metadata:
{"docstring": "", "end_line": 495, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/multiple-choice/run_swag.py", "name": "_mp_fn", "start_line": 493, "type": "function", "chroma:document": "CODE:\n\ndef _mp_fn(index):\n    # For xla_spawn (TPUs)\n\n\nSUMMARY:\n**Summary:**\n\n `_mp_fn` is an inner function that calls `main()` when utilizing XLA Spawn (TPUs), without handling alternative execution paths.\n\n**Key Details:**\n\n- Primary purpose: Execute `main()` on TPUs using XLA Spawn.\n- Method signature: `_mp_fn(index)`, no explicit parameters or return types mentioned.\n- Notable dependency:\n  - Utilizes `xla_spawn` for TPUs, a library provided by Google's XLA (Accelerated Linear Algebra) project."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py:197-211

metadata:
{"docstring": "# We use torchvision for faster image pre-processing. The transforms are implemented as nn.Module,\n# so we jit it to be faster.", "end_line": 211, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py", "name": "Transform", "start_line": 197, "type": "class", "chroma:document": "CODE:\n# so we jit it to be faster.\nclass Transform(torch.nn.Module):\n    def __init__(self, image_size, mean, std):\n        super().__init__()\n        self.transforms = torch.nn.Sequential(\n            Resize([image_size], interpolation=InterpolationMode.BICUBIC),\n            CenterCrop(image_size),\n            ConvertImageDtype(torch.float),\n            Normalize(mean, std),\n        )\n\n    def forward(self, x) -> torch.Tensor:\n        \"\"\"`x` should be an instance of `PIL.Image.Image`\"\"\"\n        with torch.no_grad():\n            x = self.transforms(x)\n\n\nSUMMARY:\n**Summary**\n\n`Transform` is a PyTorch module for image preprocessing, which applies sequence of transformations to input image `x`. It constructs a sequential pipeline using `torch.nn.Sequential`, including:\n\n*   resizing the image via `Resize`\n*   center cropping the image via `CenterCrop`\n*   converting image data type via `ConvertImageDtype` to float\n*   normalizing image values via `Normalize`\n\nThe transform pipeline is optimized for faster processing using `jit`.\n\n**Key Implementation Details**\n\n*   Input parameter types: `image_size`, `mean`, and `std`\n*   Output type: `torch.Tensor`\n*   Notable transformations:\n    *   `Resize` with interpolation mode `BICUBIC`\n    *   `CenterCrop` for image size match\n    *   `ConvertImageDtype` to float data type\n    *   `Normalize` with provided mean and std values\n\n**Notable Dependencies**\n\n*   PyTorch (`torch`)\n*   PIL (Python Imaging Library) (`PIL.Image`)"}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py:214-223

metadata:
{"docstring": "", "end_line": 223, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py", "name": "collate_fn", "start_line": 214, "type": "function", "chroma:document": "CODE:\n\ndef collate_fn(examples):\n    pixel_values = torch.stack([example[\"pixel_values\"] for example in examples])\n    input_ids = torch.tensor([example[\"input_ids\"] for example in examples], dtype=torch.long)\n    attention_mask = torch.tensor([example[\"attention_mask\"] for example in examples], dtype=torch.long)\n    return {\n        \"pixel_values\": pixel_values,\n        \"input_ids\": input_ids,\n        \"attention_mask\": attention_mask,\n        \"return_loss\": True,\n\n\nSUMMARY:\n**Function Summary: `collate_fn`**\n\n**Purpose and Functionality:** Defines a function to collate and preprocess input data for training, aggregating pixel values, input IDs, and attention masks.\n\n**Implementation Details:**\n\n* Prepends to each tensor data using stack on pixel values.\n* Creates tensors from input data (input_ids and attention_mask) with specified data types (long).\n\n**Notable Dependencies/Requirements:**\n\n* `torch`: required for tensor operations\n\n**Output:** A dictionary containing the preprocessed input data (`pixel_values`, `input_ids`, `attention_mask`) with a boolean flag indicating that returns loss (`\"return_loss\": True`)."}


created_at: 2025-02-05 09:13:38
topic: persistent://default/default/d7f79fe7-4053-4e56-8c53-540c743a4d8b
encoding: FLOAT32
id:
/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py:226-527

metadata:
{"docstring": "", "end_line": 527, "file_path": "/home/gaby/projects/2025/transformers-4.48.1/examples/pytorch/contrastive-image-text/run_clip.py", "name": "main", "start_line": 226, "type": "function", "chroma:document": "CODE:\n\ndef main():\n    # 1. Parse input arguments\n    # See all possible arguments in src/transformers/training_args.py\n    # or by passing the --help flag to this script.\n    # We now keep distinct sets of args, for a cleaner separation of concerns.\n\n    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))\n    if len(sys.argv) == 2 and sys.argv[1].endswith(\".json\"):\n        # If we pass only one argument to the script and it's the path to a json file,\n        # let's parse it to get our arguments.\n        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))\n    else:\n        model_args, data_args, training_args = parser.parse_args_into_dataclasses()\n\n    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The\n    # information sent is the one passed as arguments along with your Python/PyTorch versions.\n    send_example_telemetry(\"run_clip\", model_args, data_args)\n\n    # 2. Setup logging\n    logging.basicConfig(\n        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n        datefmt=\"%m/%d/%Y %H:%M:%S\",\n        handlers=[logging.StreamHandler(sys.stdout)],\n    )\n\n    if training_args.should_log:\n        # The default of training_args.log_level is passive, so we set log level at info here to have that default.\n        transformers.utils.logging.set_verbosity_info()\n\n    log_level = training_args.get_process_log_level()\n    logger.setLevel(log_level)\n    transformers.utils.logging.set_verbosity(log_level)\n    transformers.utils.logging.enable_default_handler()\n    transformers.utils.logging.enable_explicit_format()\n\n    # Log on each process the small summary:\n    logger.warning(\n        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, \"\n        + f\"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}\"\n    )\n    logger.info(f\"Training/evaluation parameters {training_args}\")\n\n    # 3. Detecting last checkpoint and eventually continue from last checkpoint\n    last_checkpoint = None\n    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:\n        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n            raise ValueError(\n                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n                \"Use --overwrite_output_dir to overcome.\"\n            )\n        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:\n            logger.info(\n                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n            )\n\n    # 4. Load dataset\n    # Get the datasets: you can either provide your own CSV/JSON training and evaluation files (see below)\n    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/\n    # (the dataset will be downloaded automatically from the datasets Hub).\n    #\n    # For CSV/JSON files this script will use the first column for the full image path and the second column for the\n    # captions (unless you specify column names for this with the `image_column` and `caption_column` arguments).\n    #\n    if data_args.dataset_name is not None:\n        # Downloading and loading a dataset from the hub.\n        dataset = load_dataset(\n            data_args.dataset_name,\n            data_args.dataset_config_name,\n            cache_dir=model_args.cache_dir,\n            keep_in_memory=False,\n            data_dir=data_args.data_dir,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        data_files = {}\n        if data_args.train_file is not None:\n            data_files[\"train\"] = data_args.train_file\n            extension = data_args.train_file.split(\".\")[-1]\n        if data_args.validation_file is not None:\n            data_files[\"validation\"] = data_args.validation_file\n            extension = data_args.validation_file.split(\".\")[-1]\n        dataset = load_dataset(\n            extension,\n            data_files=data_files,\n            cache_dir=model_args.cache_dir,\n            token=model_args.token,\n        )\n    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at\n    # https://huggingface.co/docs/datasets/loading_datasets.\n\n    # 5. Load pretrained model, tokenizer, and image processor\n    if model_args.tokenizer_name:\n        tokenizer = AutoTokenizer.from_pretrained(\n            model_args.tokenizer_name,\n            cache_dir=model_args.cache_dir,\n            use_fast=model_args.use_fast_tokenizer,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    elif model_args.model_name_or_path:\n        tokenizer = AutoTokenizer.from_pretrained(\n            model_args.model_name_or_path,\n            cache_dir=model_args.cache_dir,\n            use_fast=model_args.use_fast_tokenizer,\n            token=model_args.token,\n            trust_remote_code=model_args.trust_remote_code,\n        )\n    else:\n        raise ValueError(\n            \"You are instantiating a new tokenizer from scratch. This is not supported by this script. \"\n            \"You can do it from another script, save it, and load it from here, using --tokenizer_name.\"\n        )\n\n    # Load image_processor, in this script we only use this to get the mean and std for normalization.\n    image_processor = AutoImageProcessor.from_pretrained(\n        model_args.image_processor_name or model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n\n    model = AutoModel.from_pretrained(\n        model_args.model_name_or_path,\n        cache_dir=model_args.cache_dir,\n        revision=model_args.model_revision,\n        token=model_args.token,\n        trust_remote_code=model_args.trust_remote_code,\n    )\n    config = model.config\n\n    def _freeze_params(module):\n        for param in module.parameters():\n            param.requires_grad = False\n\n    if model_args.freeze_vision_model:\n        _freeze_params(model.vision_model)\n\n    if model_args.freeze_text_model:\n        _freeze_params(model.text_model)\n\n    # set seed for torch dataloaders\n    set_seed(training_args.seed)\n\n    # Preprocessing the datasets.\n    # We need to tokenize inputs and targets.\n    if training_args.do_train:\n        column_names = dataset[\"train\"].column_names\n    elif training_args.do_eval:\n        column_names = dataset[\"validation\"].column_names\n    else:\n        logger.info(\"There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.\")\n        return\n\n    # 6. Get the column names for input/target.\n    dataset_columns = dataset_name_mapping.get(data_args.dataset_name, None)\n    if data_args.image_column is None:\n        image_column = dataset_columns[0] if dataset_columns is not None else column_names[0]\n    else:\n        image_column = data_args.image_column\n        if image_column not in column_names:\n            raise ValueError(\n                f\"--image_column' value '{data_args.image_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n    if data_args.caption_column is None:\n        caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1]\n    else:\n        caption_column = data_args.caption_column\n        if caption_column not in column_names:\n            raise ValueError(\n                f\"--caption_column' value '{data_args.caption_column}' needs to be one of: {', '.join(column_names)}\"\n            )\n\n    # 7. Preprocessing the datasets.\n    # Initialize torchvision transforms and jit it for faster processing.\n    image_transformations = Transform(\n        config.vision_config.image_size, image_processor.image_mean, image_processor.image_std\n    )\n    image_transformations = torch.jit.script(image_transformations)\n\n    # Preprocessing the datasets.\n    # We need to tokenize input captions and transform the images.\n    def tokenize_captions(examples):\n        captions = list(examples[caption_column])\n        text_inputs = tokenizer(captions, max_length=data_args.max_seq_length, padding=\"max_length\", truncation=True)\n        examples[\"input_ids\"] = text_inputs.input_ids\n        examples[\"attention_mask\"] = text_inputs.attention_mask\n        return examples\n\n    def transform_images(examples):\n        images = [read_image(image_file, mode=ImageReadMode.RGB) for image_file in examples[image_column]]\n        examples[\"pixel_values\"] = [image_transformations(image) for image in images]\n        return examples\n\n    def filter_corrupt_images(examples):\n        \"\"\"remove problematic images\"\"\"\n        valid_images = []\n        for image_file in examples[image_column]:\n            try:\n                Image.open(image_file)\n                valid_images.append(True)\n            except Exception:\n                valid_images.append(False)\n        return valid_images\n\n    if training_args.do_train:\n        if \"train\" not in dataset:\n            raise ValueError(\"--do_train requires a train dataset\")\n        train_dataset = dataset[\"train\"]\n        if data_args.max_train_samples is not None:\n            max_train_samples = min(len(train_dataset), data_args.max_train_samples)\n            train_dataset = train_dataset.select(range(max_train_samples))\n\n        train_dataset = train_dataset.filter(\n            filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers\n        )\n        train_dataset = train_dataset.map(\n            function=tokenize_captions,\n            batched=True,\n            remove_columns=[col for col in column_names if col != image_column],\n            num_proc=data_args.preprocessing_num_workers,\n            load_from_cache_file=not data_args.overwrite_cache,\n            desc=\"Running tokenizer on train dataset\",\n        )\n\n        # Transform images on the fly as doing it on the whole dataset takes too much time.\n        train_dataset.set_transform(transform_images)\n\n    if training_args.do_eval:\n        if \"validation\" not in dataset:\n            raise ValueError(\"--do_eval requires a train validation\")\n        eval_dataset = dataset[\"validation\"]\n        if data_args.max_eval_samples is not None:\n            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)\n            eval_dataset = eval_dataset.select(range(max_eval_samples))\n\n        eval_dataset = eval_dataset.filter(\n            filter_corrupt_images, batched=True, num_proc=data_args.preprocessing_num_workers\n        )\n        eval_dataset = eval_dataset.map(\n            function=tokenize_captions,\n            batched=True,\n            num_proc=data_args.preprocessing_num_workers,\n            remove_columns=[col for col in column_names if col != image_column],\n            load_from_cache_file=not data_args.overwrite_cache,\n            desc=\"Running tokenizer on validation dataset\",\n        )\n\n        # Transform images on the fly as doing it on the whole dataset takes too much time.\n        eval_dataset.set_transform(transform_images)\n\n    # 8. Initialize our trainer\n    trainer = Trainer(\n        model=model,\n        args=training_args,\n        train_dataset=train_dataset if training_args.do_train else None,\n        eval_dataset=eval_dataset if training_args.do_eval else None,\n        data_collator=collate_fn,\n    )\n\n    # 9. Training\n    if training_args.do_train:\n        checkpoint = None\n        if training_args.resume_from_checkpoint is not None:\n            checkpoint = training_args.resume_from_checkpoint\n        elif last_checkpoint is not None:\n            checkpoint = last_checkpoint\n        train_result = trainer.train(resume_from_checkpoint=checkpoint)\n        trainer.save_model()\n        tokenizer.save_pretrained(training_args.output_dir)\n        image_processor.save_pretrained(training_args.output_dir)\n        trainer.log_metrics(\"train\", train_result.metrics)\n        trainer.save_metrics(\"train\", train_result.metrics)\n        trainer.save_state()\n\n    # 10. Evaluation\n    if training_args.do_eval:\n        metrics = trainer.evaluate()\n        trainer.log_metrics(\"eval\", metrics)\n        trainer.save_metrics(\"eval\", metrics)\n\n    # 11. Write Training Stats and push to hub.\n    finetuned_from = model_args.model_name_or_path\n    # If from a local directory, don't set `finetuned_from` as this is required to be a valid repo. id on the Hub.\n    if os.path.isdir(finetuned_from):\n        finetuned_from = None\n    kwargs = {\"finetuned_from\": finetuned_from, \"tasks\": \"contrastive-image-text-modeling\"}\n    if data_args.dataset_name is not None:\n        kwargs[\"dataset_tags\"] = data_args.dataset_name\n        if data_args.dataset_config_name is not None:\n            kwargs[\"dataset_args\"] = data_args.dataset_config_name\n            kwargs[\"dataset\"] = f\"{data_args.dataset_name} {data_args.dataset_config_name}\"\n        else:\n            kwargs[\"dataset\"] = data_args.dataset_name\n\n    if training_args.push_to_hub:\n        trainer.push_to_hub(**kwargs)\n    else:\n\n\nSUMMARY:\nHere is a concise summary of the code:\n\nThe code trains a Contrastive Image-Text Modeling (CITM) model using the Hugging Face Transformers library. The CITM model is trained to learn representations from pairs of images and text captions. \n\nThe training process involves:\n\n1. Preprocessing: removing corrupt images, tokenizing text inputs, transforming images on the fly.\n2. Training: using the Trainer class from Hugging Face to train the model with a batched dataset (if `do_train` is True).\n3. Evaluation: evaluating the trained model on a separate validation set (if `do_eval` is True) and saving the results.\n4. Saving models, tokenizers, and image processors to disk and pushing them to the Hugging Face model hub if `push_to_hub` is True.\n\nThe code makes use of utility functions from Hugging Face such as `collate_fn`, `finetuned_from` which are passed as arguments to various Trainer methods.\n\nThis code can be used to train a pre-trained CITM model on a specific dataset and fine-tune it for a specific task. \n\nHere's some key information:\n\n- Training strategy: Training is configured using the Hugging Face Trainer.\n- Loss function and metrics: Not explicitly specified, but common loss functions and metrics may include cross-entropy loss or top-1 accuracy.\n- Dataset: Pre-trained dataset (e.g., COCO) used for training and evaluation.\n- Parameters:\n  - `data_args`: Command-line arguments for model pretraining.\n  - `training_args`: Training parameters such as batch size, number of epochs, etc. \n  - `model`: Instance of the Contrastive Image-Text Modeling (CITM) model.\n\nThe code is written in Python using several Hugging Face libraries, including Transformers, Trainer, and datasets."}


