# embedding_metadata

key: chroma:document
string_value:
CODE:

def main():
    # region Argument Parsing
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/PyTorch versions.
    send_example_telemetry("run_clm", model_args, data_args, framework="tensorflow")

    # Sanity checks
    if data_args.dataset_name is None and data_args.train_file is None and data_args.validation_file is None:
        raise ValueError("Need either a dataset name or a training/validation file.")
    else:
        if data_args.train_file is not None:
            extension = data_args.train_file.split(".")[-1]
            assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, json or txt file."
        if data_args.validation_file is not None:
            extension = data_args.validation_file.split(".")[-1]
            assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, json or txt file."

    if training_args.output_dir is not None:
        training_args.output_dir = Path(training_args.output_dir)
        os.makedirs(training_args.output_dir, exist_ok=True)
    # endregion

    # region Checkpoints
    # Detecting last checkpoint.
    checkpoint = None
    if len(os.listdir(training_args.output_dir)) > 0 and not training_args.overwrite_output_dir:
        config_path = training_args.output_dir / CONFIG_NAME
        weights_path = training_args.output_dir / TF2_WEIGHTS_NAME
        if config_path.is_file() and weights_path.is_file():
            checkpoint = training_args.output_dir
            logger.info(
                f"Checkpoint detected, resuming training from checkpoint in {training_args.output_dir}. To avoid this"
                " behavior, change the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )
        else:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to continue regardless."
            )

    # endregion

    # region Setup logging
    # accelerator.is_local_main_process is only True for one process per machine.
    logger.setLevel(logging.INFO)
    datasets.utils.logging.set_verbosity_warning()
    transformers.utils.logging.set_verbosity_info()
    # endregion

    # If passed along, set the training seed now.
    if training_args.seed is not None:
        set_seed(training_args.seed)

    # region Load datasets
    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
    # (the dataset will be downloaded automatically from the datasets Hub).
    #
    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
    # 'text' is found. You can easily tweak this behavior (see below).
    #
    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
    # download the dataset.
    if data_args.dataset_name is not None:
        # Downloading and loading a dataset from the hub.
        raw_datasets = load_dataset(
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
            token=model_args.token,
            trust_remote_code=model_args.trust_remote_code,
        )
        if "validation" not in raw_datasets.keys():
            raw_datasets["validation"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
                token=model_args.token,
                trust_remote_code=model_args.trust_remote_code,
            )
            raw_datasets["train"] = load_dataset(
                data_args.dataset_name,
                data_args.dataset_config_name,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
                token=model_args.token,
                trust_remote_code=model_args.trust_remote_code,
            )
    else:
        data_files = {}
        dataset_args = {}
        if data_args.train_file is not None:
            data_files["train"] = data_args.train_file
        if data_args.validation_file is not None:
            data_files["validation"] = data_args.validation_file
        extension = (
            data_args.train_file.split(".")[-1]
            if data_args.train_file is not None
            else data_args.validation_file.split(".")[-1]
        )
        if extension == "txt":
            extension = "text"
            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
        raw_datasets = load_dataset(
            extension,
            data_files=data_files,
            cache_dir=model_args.cache_dir,
            token=model_args.token,
            **dataset_args,
        )
        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
        if "validation" not in raw_datasets.keys():
            raw_datasets["validation"] = load_dataset(
                extension,
                data_files=data_files,
                split=f"train[:{data_args.validation_split_percentage}%]",
                cache_dir=model_args.cache_dir,
                token=model_args.token,
                **dataset_args,
            )
            raw_datasets["train"] = load_dataset(
                extension,
                data_files=data_files,
                split=f"train[{data_args.validation_split_percentage}%:]",
                cache_dir=model_args.cache_dir,
                token=model_args.token,
                **dataset_args,
            )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.
    # endregion

    # region Load pretrained model and tokenizer
    #
    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.
    if model_args.config_name:
        config = AutoConfig.from_pretrained(
            model_args.config_name,
            token=model_args.token,
            trust_remote_code=model_args.trust_remote_code,
        )
    elif model_args.model_name_or_path:
        config = AutoConfig.from_pretrained(
            model_args.model_name_or_path, token=model_args.token, trust_remote_code=model_args.trust_remote_code
        )
    else:
        config = CONFIG_MAPPING[model_args.model_type]()
        logger.warning("You are instantiating a new config instance from scratch.")

    if model_args.tokenizer_name:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.tokenizer_name, token=model_args.token, trust_remote_code=model_args.trust_remote_code
        )
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path, token=model_args.token, trust_remote_code=model_args.trust_remote_code
        )
    else:
        raise ValueError(
            "You are instantiating a new tokenizer from scratch. This is not supported by this script. "
            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
        )
    # endregion

    # region Dataset preprocessing
    # First we tokenize all the texts.
    column_names = raw_datasets["train"].column_names
    text_column_name = "text" if "text" in column_names else column_names[0]

    def tokenize_function(examples):
        return tokenizer(examples[text_column_name])

    tokenized_datasets = raw_datasets.map(
        tokenize_function,
        batched=True,
        num_proc=data_args.preprocessing_num_workers,
        remove_columns=column_names,
        load_from_cache_file=not data_args.overwrite_cache,
        desc="Running tokenizer on dataset",
    )

    if data_args.block_size is None:
        block_size = tokenizer.model_max_length
        if block_size > config.max_position_embeddings:
            logger.warning(
                f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
                f"Using block_size={min(1024, config.max_position_embeddings)} instead. You can change that default value by passing --block_size xxx."
            )
            block_size = min(1024, config.max_position_embeddings)
    else:
        if data_args.block_size > tokenizer.model_max_length:
            logger.warning(
                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model "
                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
            )
        block_size = min(data_args.block_size, tokenizer.model_max_length)

    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
    def group_texts(examples):
        # Concatenate all texts.
        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
        total_length = len(concatenated_examples[list(examples.keys())[0]])
        # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
        # customize this part to your needs.
        if total_length >= block_size:
            total_length = (total_length // block_size) * block_size
        # Split by chunks of max_len.
        result = {
            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
            for k, t in concatenated_examples.items()
        }
        result["labels"] = result["input_ids"].copy()
        return result

    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
    # to preprocess.
    #
    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
    # https://huggingface.co/docs/datasets/process#map

    lm_datasets = tokenized_datasets.map(
        group_texts,
        batched=True,
        num_proc=data_args.preprocessing_num_workers,
        load_from_cache_file=not data_args.overwrite_cache,
        desc=f"Grouping texts in chunks of {block_size}",
    )

    train_dataset = lm_datasets["train"]
    if data_args.validation_file is not None:
        eval_dataset = lm_datasets["validation"]
    else:
        logger.info(
            f"Validation file not found: using {data_args.validation_split_percentage}% of the dataset as validation"
            " as provided in data_args"
        )
        train_indices, val_indices = train_test_split(
            list(range(len(train_dataset))), test_size=data_args.validation_split_percentage / 100
        )

        eval_dataset = train_dataset.select(val_indices)
        train_dataset = train_dataset.select(train_indices)

    if data_args.max_train_samples is not None:
        max_train_samples = min(len(train_dataset), data_args.max_train_samples)
        train_dataset = train_dataset.select(range(max_train_samples))
    if data_args.max_eval_samples is not None:
        max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
        eval_dataset = eval_dataset.select(range(max_eval_samples))

    # Log a few random samples from the training set:
    for index in random.sample(range(len(train_dataset)), min(3, len(train_dataset))):
        logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")
    # endregion

    with training_args.strategy.scope():
        # region Prepare model
        if checkpoint is not None:
            model = TFAutoModelForCausalLM.from_pretrained(
                checkpoint, config=config, token=model_args.token, trust_remote_code=model_args.trust_remote_code
            )
        elif model_args.model_name_or_path:
            model = TFAutoModelForCausalLM.from_pretrained(
                model_args.model_name_or_path,
                config=config,
                token=model_args.token,
                trust_remote_code=model_args.trust_remote_code,
            )
        else:
            logger.info("Training new model from scratch")
            model = TFAutoModelForCausalLM.from_config(
                config, token=model_args.token, trust_remote_code=model_args.trust_remote_code
            )

        # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
        # on a small vocab and want a smaller embedding size, remove this test.
        embeddings = model.get_input_embeddings()

        # Matt: This is a temporary workaround as we transition our models to exclusively using Keras embeddings.
        #       As soon as the transition is complete, all embeddings should be keras.Embeddings layers, and
        #       the weights will always be in embeddings.embeddings.
        if hasattr(embeddings, "embeddings"):
            embedding_size = embeddings.embeddings.shape[0]
        else:
            embedding_size = embeddings.weight.shape[0]
        if len(tokenizer) > embedding_size:
            model.resize_token_embeddings(len(tokenizer))
        # endregion

        # region TF Dataset preparation
        num_replicas = training_args.strategy.num_replicas_in_sync
        options = tf.data.Options()
        options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF

        # model.prepare_tf_dataset() wraps a Hugging Face dataset in a tf.data.Dataset which is ready to use in
        # training. This is the recommended way to use a Hugging Face dataset when training with Keras. You can also
        # use the lower-level dataset.to_tf_dataset() method, but you will have to specify things like column names
        # yourself if you use this method, whereas they are automatically inferred from the model input names when
        # using model.prepare_tf_dataset()
        # For more info see the docs:
        # https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset
        # https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset

        tf_train_dataset = model.prepare_tf_dataset(
            train_dataset,
            shuffle=True,
            batch_size=num_replicas * training_args.per_device_train_batch_size,
        ).with_options(options)

        tf_eval_dataset = model.prepare_tf_dataset(
            eval_dataset,
            shuffle=False,
            batch_size=num_replicas * training_args.per_device_eval_batch_size,
            drop_remainder=True,
        ).with_options(options)
        # endregion

        # region Optimizer and loss
        num_train_steps = len(tf_train_dataset) * int(training_args.num_train_epochs)
        if training_args.warmup_steps > 0:
            num_warmup_steps = training_args.warmup_steps
        elif training_args.warmup_ratio > 0:
            num_warmup_steps = int(num_train_steps * training_args.warmup_ratio)
        else:
            num_warmup_steps = 0

        # Bias and layernorm weights are automatically excluded from the decay
        optimizer, lr_schedule = create_optimizer(
            init_lr=training_args.learning_rate,
            num_train_steps=num_train_steps,
            num_warmup_steps=num_warmup_steps,
            adam_beta1=training_args.adam_beta1,
            adam_beta2=training_args.adam_beta2,
            adam_epsilon=training_args.adam_epsilon,
            weight_decay_rate=training_args.weight_decay,
            adam_global_clipnorm=training_args.max_grad_norm,
        )

        # Transformers models compute the right loss for their task by default when labels are passed, and will
        # use this for training unless you specify your own loss function in compile().
        model.compile(optimizer=optimizer, jit_compile=training_args.xla)
        # endregion

        # region Preparing push_to_hub and model card
        push_to_hub_model_id = training_args.push_to_hub_model_id
        model_name = model_args.model_name_or_path.split("/")[-1]
        if not push_to_hub_model_id:
            if data_args.dataset_name is not None:
                push_to_hub_model_id = f"{model_name}-finetuned-{data_args.dataset_name}"
            else:
                push_to_hub_model_id = f"{model_name}-finetuned-clm"

        model_card_kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
        if data_args.dataset_name is not None:
            model_card_kwargs["dataset_tags"] = data_args.dataset_name
            if data_args.dataset_config_name is not None:
                model_card_kwargs["dataset_args"] = data_args.dataset_config_name
                model_card_kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
            else:
                model_card_kwargs["dataset"] = data_args.dataset_name

        if training_args.push_to_hub:
            callbacks = [
                PushToHubCallback(
                    output_dir=training_args.output_dir,
                    hub_model_id=push_to_hub_model_id,
                    hub_token=training_args.push_to_hub_token,
                    tokenizer=tokenizer,
                    **model_card_kwargs,
                )
            ]
        else:
            callbacks = []
        # endregion

        # region Training and validation
        logger.info("***** Running training *****")
        logger.info(f"  Num examples = {len(train_dataset)}")
        logger.info(f"  Num Epochs = {training_args.num_train_epochs}")
        logger.info(f"  Instantaneous batch size per device = {training_args.per_device_train_batch_size}")
        logger.info(f"  Total train batch size = {training_args.per_device_train_batch_size * num_replicas}")

        # For long training runs, you may wish to use the PushToHub() callback here to save intermediate checkpoints
        # to the Hugging Face Hub rather than just pushing the finished model.
        # See https://huggingface.co/docs/transformers/main_classes/keras_callbacks#transformers.PushToHubCallback

        history = model.fit(
            tf_train_dataset,
            validation_data=tf_eval_dataset,
            epochs=int(training_args.num_train_epochs),
            callbacks=callbacks,
        )
        train_loss = history.history["loss"][-1]
        try:
            train_perplexity = math.exp(train_loss)
        except OverflowError:
            train_perplexity = math.inf
        logger.info(f"  Final train loss: {train_loss:.3f}")
        logger.info(f"  Final train perplexity: {train_perplexity:.3f}")
        validation_loss = history.history["val_loss"][-1]
        try:
            validation_perplexity = math.exp(validation_loss)
        except OverflowError:
            validation_perplexity = math.inf
        logger.info(f"  Final validation loss: {validation_loss:.3f}")
        logger.info(f"  Final validation perplexity: {validation_perplexity:.3f}")

        if training_args.output_dir is not None:
            output_eval_file = os.path.join(training_args.output_dir, "all_results.json")
            results_dict = {}
            results_dict["train_loss"] = train_loss
            results_dict["train_perplexity"] = train_perplexity
            results_dict["eval_loss"] = validation_loss
            results_dict["eval_perplexity"] = validation_perplexity
            with open(output_eval_file, "w") as writer:
                writer.write(json.dumps(results_dict))
        # endregion

    if training_args.output_dir is not None and not training_args.push_to_hub:
        # If we're not pushing to hub, at least save a local copy when we're done


SUMMARY:
Here's a concise summary of the code:

**Training a Text Generation Model**

This is a script that trains a text generation model using the Hugging Face Transformers library. The model takes in a set of input data and generates text based on the task provided.

**Key Steps:**

1. **Data Preparation**: Load the dataset into the model.
2. **Model Initialization**: Initialize the model with the given parameters (e.g., model name, learning rate).
3. **Loss Function and Optimizer**: Define the loss function and optimizer for training.
4. **Model Compilation**: Compile the model with the chosen optimizer and loss function.
5. **Training**: Train the model on the dataset using the defined hyperparameters.
6. **Validation**: Validate the model's performance on a separate validation set.

**Key Hyperparameters:**

* Learning rate
* Number of epochs
* Batch size
* Regularization parameters (e.g., weight decay, Adam beta-values)

**Additional Features:**

* Pushing the final trained model to the Hugging Face Hub for sharing.
* Saving the evaluation results as a JSON file.

Overall, this script provides a flexible framework for training and fine-tuning text generation models using Transformers.


key: file_path
string_value:
/home/gaby/projects/2025/transformers-4.48.1/examples/tensorflow/image-classification/run_image_classification.py


key: chroma:document
string_value:
CODE:

def pil_loader(path: str):
    with open(path, "rb") as f:
        im = Image.open(f)


SUMMARY:
**pil_loader Function Summary**

* **Purpose:** Loads and converts an image to RGB format using the Python Imaging Library (PIL).

**Function Details:**

* Reads an image from the specified `path` file in binary mode (`"rb"`).
* Uses PIL's `Image.open()` function to load the image.
* Converts the loaded image to the "RGB" color space using the `convert()` method.

**External Dependencies:**
- Python Imaging Library (PIL)

No additional interfaces or classes are implemented or inherited by this function.


key: chroma:document
string_value:
CODE:

def center_crop(image, size):
    size = (size, size) if isinstance(size, int) else size
    orig_height, orig_width, _ = image.shape
    crop_height, crop_width = size
    top = (orig_height - orig_width) // 2
    left = (orig_width - crop_width) // 2
    image = tf.image.crop_to_bounding_box(image, top, left, crop_height, crop_width)


SUMMARY:
**Summary:**

This function `(center_crop)` takes an `image` and optional `size` parameters. It centers a cropped region to the specified size within the original image.

**Implementation Details:**

- **Crop Region Calculation**: The top-left coordinates are calculated based on the difference between the image dimensions and the crop size.
- **Image Cropping**: The `tf.image.crop_to_bounding_box` function is used to extract the centered cropped region from the original image.

**Notable Dependencies/Requirements:**

- `tf` (TensorFlow) for image processing functionality
- `(size, size)` or exact integer size as input, defaulting to square if an integer is provided


key: docstring
string_value:
# Numpy and TensorFlow compatible version of PyTorch RandomResizedCrop. Code adapted from:
# https://pytorch.org/vision/main/_modules/torchvision/transforms/transforms.html#RandomResizedCrop


key: chroma:document
string_value:
CODE:
# https://pytorch.org/vision/main/_modules/torchvision/transforms/transforms.html#RandomResizedCrop
def random_crop(image, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)):
    height, width, _ = image.shape
    area = height * width
    log_ratio = np.log(ratio)
    for _ in range(10):
        target_area = np.random.uniform(*scale) * area
        aspect_ratio = np.exp(np.random.uniform(*log_ratio))
        w = int(round(np.sqrt(target_area * aspect_ratio)))
        h = int(round(np.sqrt(target_area / aspect_ratio)))
        if 0 < w <= width and 0 < h <= height:
            i = np.random.randint(0, height - h + 1)
            j = np.random.randint(0, width - w + 1)
            return image[i : i + h, j : j + w, :]

    # Fallback to central crop
    in_ratio = float(width) / float(height)
    w = width if in_ratio < min(ratio) else int(round(height * max(ratio)))
    h = height if in_ratio > max(ratio) else int(round(width / min(ratio)))
    i = (height - h) // 2
    j = (width - w) // 2


SUMMARY:
**Concise Summary**

```python
def random_crop(image, 
               scale=(0.08, 1.0), 
               ratio=(3.0 / 4.0, 4.0 / 3.0)):
    """
    Returns a randomly resized crop of the input image.
    
    Args:
        image: Input image tensor.
        scale: Tuple of (min_scale, max_scale) for area scaling.
        ratio: Tuple of (width_ratio, height_ratio) for aspect ratio.
    
    Returns:
        A cropped image tensor.
    """
```

**Implementation Details**

* The function generates a random crop by iterating 10 times through different scales and ratios to find a valid crop.
* On the 11th iteration, if no valid crop is found, it falls back to a central crop using a median aspect ratio.

**Dependencies**

* `numpy` for numerical computations
* No explicit dependencies on TensorFlow (TF) since this code snippet only defines the `random_crop` function.

**Key Insights**

* The function uses both area scaling and aspect ratio scaling for random cropping.
* It employs a fallback to central crop in case no valid crop is found after 10 iterations.


key: chroma:document
string_value:
CODE:

def random_resized_crop(image, size, scale=(0.08, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)):
    size = (size, size) if isinstance(size, int) else size
    image = random_crop(image, scale, ratio)
    image = tf.image.resize(image, size)


SUMMARY:
**Summary:**

This function (`random_resized_crop`) generates a randomized cropped image with a specified `size`, while maintaining its aspect ratio.

**Key Details:**

* Resizes an input `image` to a random size within the specified `scale` and `ratio`.
* Utilizes TensorFlow's `tf.image.resize` to resize the image.

**Implementation Parameters:**

* `size`: The target size of the output image (int or tuple).
* `scale`: A scaling factor for randomly resizing the image (tuple).
* `ratio`: The aspect ratio constraint for the resized image (tuple).
* `random_crop`: Utilized function from TensorFlow for cropping (not shown in this code snippet).

**Notable Dependencies:**

* TensorFlow (`tf`): For image processing and randomization utilities.
* `random_crop`: A custom or external function implemented by the library.


key: chroma:document
string_value:
CODE:

def main():
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.
    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments))
    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
        # If we pass only one argument to the script and it's the path to a json file,
        # let's parse it to get our arguments.
        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
    else:
        model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
    # information sent is the one passed as arguments along with your Python/TensorFlow versions.
    send_example_telemetry("run_image_classification", model_args, data_args, framework="tensorflow")

    # Checkpoints. Find the checkpoint the use when loading the model.
    checkpoint = None
    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
        checkpoint = get_last_checkpoint(training_args.output_dir)
        if checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
            raise ValueError(
                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
                "Use --overwrite_output_dir to overcome."
            )
        elif checkpoint is not None and training_args.resume_from_checkpoint is None:
            logger.info(
                f"Checkpoint detected, resuming training at {checkpoint}. To avoid this behavior, change "
                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
            )

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        handlers=[logging.StreamHandler(sys.stdout)],
    )
    log_level = training_args.get_process_log_level()
    logger.setLevel(log_level)

    # Set the verbosity to info of the Transformers logger (on main process only):
    if is_main_process(training_args.local_rank):
        transformers.utils.logging.set_verbosity_info()
        transformers.utils.logging.enable_default_handler()
        transformers.utils.logging.enable_explicit_format()
    logger.info(f"Training/evaluation parameters {training_args}")

    # region Dataset and labels
    # Set seed before initializing model.
    set_seed(training_args.seed)

    # Initialize our dataset and prepare it for the 'image-classification' task.
    if data_args.dataset_name is not None:
        dataset = load_dataset(
            data_args.dataset_name,
            data_args.dataset_config_name,
            cache_dir=model_args.cache_dir,
            token=model_args.token,
            trust_remote_code=model_args.trust_remote_code,
        )
    else:
        data_files = {}
        if data_args.train_dir is not None:
            data_files["train"] = os.path.join(data_args.train_dir, "**")
        if data_args.validation_dir is not None:
            data_files["validation"] = os.path.join(data_args.validation_dir, "**")
        dataset = load_dataset(
            "imagefolder",
            data_files=data_files,
            cache_dir=model_args.cache_dir,
        )
    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
    # https://huggingface.co/docs/datasets/loading_datasets.

    # Prepare label mappings.
    # We'll include these in the model's config to get human readable labels in the Inference API.
    labels = dataset["train"].features["labels"].names
    label2id, id2label = {}, {}
    for i, label in enumerate(labels):
        label2id[label] = str(i)
        id2label[str(i)] = label

    # Load model image processor and configuration
    config = AutoConfig.from_pretrained(
        model_args.config_name or model_args.model_name_or_path,
        num_labels=len(labels),
        label2id=label2id,
        id2label=id2label,
        finetuning_task="image-classification",
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
    )
    image_processor = AutoImageProcessor.from_pretrained(
        model_args.image_processor_name or model_args.model_name_or_path,
        cache_dir=model_args.cache_dir,
        revision=model_args.model_revision,
        token=model_args.token,
        trust_remote_code=model_args.trust_remote_code,
    )

    # If we don't have a validation split, split off a percentage of train as validation.
    data_args.train_val_split = None if "validation" in dataset.keys() else data_args.train_val_split
    if isinstance(data_args.train_val_split, float) and data_args.train_val_split > 0.0:
        split = dataset["train"].train_test_split(data_args.train_val_split)
        dataset["train"] = split["train"]
        dataset["validation"] = split["test"]

    # Define our data preprocessing function. It takes an image file path as input and returns
    # Write a note describing the resizing behaviour.
    if "shortest_edge" in image_processor.size:
        # We instead set the target size as (shortest_edge, shortest_edge) to here to ensure all images are batchable.
        image_size = (image_processor.size["shortest_edge"], image_processor.size["shortest_edge"])
    else:
        image_size = (image_processor.size["height"], image_processor.size["width"])

    def _train_transforms(image):
        img_size = image_size
        image = keras.utils.img_to_array(image)
        image = random_resized_crop(image, size=img_size)
        image = tf.image.random_flip_left_right(image)
        image /= 255.0
        image = (image - image_processor.image_mean) / image_processor.image_std
        image = tf.transpose(image, perm=[2, 0, 1])
        return image

    def _val_transforms(image):
        image = keras.utils.img_to_array(image)
        image = tf.image.resize(image, size=image_size)
        # image = np.array(image) # FIXME - use tf.image function
        image = center_crop(image, size=image_size)
        image /= 255.0
        image = (image - image_processor.image_mean) / image_processor.image_std
        image = tf.transpose(image, perm=[2, 0, 1])
        return image

    def train_transforms(example_batch):
        """Apply _train_transforms across a batch."""
        example_batch["pixel_values"] = [
            _train_transforms(pil_img.convert("RGB")) for pil_img in example_batch["image"]
        ]
        return example_batch

    def val_transforms(example_batch):
        """Apply _val_transforms across a batch."""
        example_batch["pixel_values"] = [_val_transforms(pil_img.convert("RGB")) for pil_img in example_batch["image"]]
        return example_batch

    train_dataset = None
    if training_args.do_train:
        if "train" not in dataset:
            raise ValueError("--do_train requires a train dataset")
        train_dataset = dataset["train"]
        if data_args.max_train_samples is not None:
            train_dataset = train_dataset.shuffle(seed=training_args.seed).select(range(data_args.max_train_samples))
        train_dataset = train_dataset.map(
            train_transforms,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    eval_dataset = None
    if training_args.do_eval:
        if "validation" not in dataset:
            raise ValueError("--do_eval requires a validation dataset")
        eval_dataset = dataset["validation"]
        if data_args.max_eval_samples is not None:
            eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
        # Set the validation transforms
        eval_dataset = eval_dataset.map(
            val_transforms,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    predict_dataset = None
    if training_args.do_predict:
        if "test" not in dataset:
            raise ValueError("--do_predict requires a test dataset")
        predict_dataset = dataset["test"]
        if data_args.max_predict_samples is not None:
            predict_dataset = predict_dataset.select(range(data_args.max_predict_samples))
        # Set the test transforms
        predict_dataset = predict_dataset.map(
            val_transforms,
            batched=True,
            num_proc=data_args.preprocessing_num_workers,
            load_from_cache_file=not data_args.overwrite_cache,
        )

    collate_fn = DefaultDataCollator(return_tensors="np")

    # Load the accuracy metric from the datasets package
    metric = evaluate.load("accuracy", cache_dir=model_args.cache_dir)

    # Define our compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
    # predictions and label_ids field) and has to return a dictionary string to float.
    def compute_metrics(p):
        """Computes accuracy on a batch of predictions"""
        logits, label_ids = p
        predictions = np.argmax(logits, axis=-1)
        metrics = metric.compute(predictions=predictions, references=label_ids)
        return metrics

    with training_args.strategy.scope():
        if checkpoint is None:
            model_path = model_args.model_name_or_path
        else:
            model_path = checkpoint

        model = TFAutoModelForImageClassification.from_pretrained(
            model_path,
            config=config,
            from_pt=bool(".bin" in model_path),
            cache_dir=model_args.cache_dir,
            revision=model_args.model_revision,
            token=model_args.token,
            trust_remote_code=model_args.trust_remote_code,
            ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,
        )
        num_replicas = training_args.strategy.num_replicas_in_sync
        total_train_batch_size = training_args.per_device_train_batch_size * num_replicas
        total_eval_batch_size = training_args.per_device_eval_batch_size * num_replicas

        dataset_options = tf.data.Options()
        dataset_options.experimental_distribute.auto_shard_policy = tf.data.experimental.AutoShardPolicy.OFF

        if training_args.do_train:
            num_train_steps = int(len(train_dataset) * training_args.num_train_epochs)
            if training_args.warmup_steps > 0:
                num_warmpup_steps = int(training_args.warmup_steps)
            elif training_args.warmup_ratio > 0:
                num_warmpup_steps = int(training_args.warmup_ratio * num_train_steps)
            else:
                num_warmpup_steps = 0

            optimizer, _ = create_optimizer(
                init_lr=training_args.learning_rate,
                num_train_steps=num_train_steps,
                num_warmup_steps=num_warmpup_steps,
                adam_beta1=training_args.adam_beta1,
                adam_beta2=training_args.adam_beta2,
                adam_epsilon=training_args.adam_epsilon,
                weight_decay_rate=training_args.weight_decay,
                adam_global_clipnorm=training_args.max_grad_norm,
            )
            # model.prepare_tf_dataset() wraps a Hugging Face dataset in a tf.data.Dataset which is ready to use in
            # training. This is the recommended way to use a Hugging Face dataset when training with Keras. You can also
            # use the lower-level dataset.to_tf_dataset() method, but you will have to specify things like column names
            # yourself if you use this method, whereas they are automatically inferred from the model input names when
            # using model.prepare_tf_dataset()
            # For more info see the docs:
            # https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.TFPreTrainedModel.prepare_tf_dataset
            # https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset
            train_dataset = model.prepare_tf_dataset(
                train_dataset,
                shuffle=True,
                batch_size=total_train_batch_size,
                collate_fn=collate_fn,
            ).with_options(dataset_options)
        else:
            optimizer = "sgd"  # Just write anything because we won't be using it

        if training_args.do_eval:
            eval_dataset = model.prepare_tf_dataset(
                eval_dataset,
                shuffle=False,
                batch_size=total_eval_batch_size,
                collate_fn=collate_fn,
            ).with_options(dataset_options)

        if training_args.do_predict:
            predict_dataset = model.prepare_tf_dataset(
                predict_dataset,
                shuffle=False,
                batch_size=total_eval_batch_size,
                collate_fn=collate_fn,
            ).with_options(dataset_options)

        # Transformers models compute the right loss for their task by default when labels are passed, and will
        # use this for training unless you specify your own loss function in compile().
        model.compile(optimizer=optimizer, jit_compile=training_args.xla, metrics=["accuracy"])

        push_to_hub_model_id = training_args.push_to_hub_model_id
        if not push_to_hub_model_id:
            model_name = model_args.model_name_or_path.split("/")[-1]
            push_to_hub_model_id = f"{model_name}-finetuned-image-classification"

        model_card_kwargs = {
            "finetuned_from": model_args.model_name_or_path,
            "tasks": "image-classification",
            "dataset": data_args.dataset_name,
            "tags": ["image-classification", "tensorflow", "vision"],
        }

        callbacks = []
        if eval_dataset is not None:
            callbacks.append(KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=eval_dataset))
        if training_args.push_to_hub:
            callbacks.append(
                PushToHubCallback(
                    output_dir=training_args.output_dir,
                    hub_model_id=push_to_hub_model_id,
                    hub_token=training_args.push_to_hub_token,
                    tokenizer=image_processor,
                    **model_card_kwargs,
                )
            )

        if training_args.do_train:
            model.fit(
                train_dataset,
                validation_data=eval_dataset,
                epochs=int(training_args.num_train_epochs),
                callbacks=callbacks,
            )

        if training_args.do_eval:
            n_eval_batches = len(eval_dataset)
            eval_predictions = model.predict(eval_dataset, steps=n_eval_batches)
            eval_labels = dataset["validation"]["labels"][: n_eval_batches * total_eval_batch_size]
            eval_metrics = compute_metrics((eval_predictions.logits, eval_labels))
            logging.info("Eval metrics:")
            for metric_name, value in eval_metrics.items():
                logging.info(f"{metric_name}: {value:.3f}")

        if training_args.output_dir is not None:
            os.makedirs(training_args.output_dir, exist_ok=True)
            with open(os.path.join(training_args.output_dir, "all_results.json"), "w") as f:
                f.write(json.dumps(eval_metrics))

        if training_args.do_predict:
            n_predict_batches = len(predict_dataset)
            test_predictions = model.predict(predict_dataset, steps=n_predict_batches)
            test_labels = dataset["validation"]["labels"][: n_predict_batches * total_eval_batch_size]
            test_metrics = compute_metrics((test_predictions.logits, test_labels))
            logging.info("Test metrics:")
            for metric_name, value in test_metrics.items():
                logging.info(f"{metric_name}: {value:.3f}")

        if training_args.output_dir is not None and not training_args.push_to_hub:
            # If we're not pushing to hub, at least save a local copy when we're done


SUMMARY:
This is a Python script that trains a Hugging Face Transformers model on an image classification task. Here's a concise summary:

1. The script sets up the training environment, including the model, optimizer, and hyperparameters.
2. It prepares the datasets for training and evaluation using `model.prepare_tf_dataset()`.
3. The model is compiled with the specified loss function, optimizer, and metrics (accuracy).
4. The model is trained on the training dataset, evaluated on the validation dataset, and predicts on the test dataset.
5. After training, the model's performance is logged, and results are saved to a JSON file if desired.

Key features:

* Uses Hugging Face Transformers for image classification
* Supports optional evaluation, prediction, and saving modes
* Allows for hyperparameter tuning and output directory customization

Note that this script assumes you have a basic understanding of PyTorch and Keras, as well as the Hugging Face Transformers library.


key: chroma:document
string_value:
CODE:

def main():
    parser = HfArgumentParser(TensorFlowBenchmarkArguments)
    benchmark_args = parser.parse_args_into_dataclasses()[0]
    benchmark = TensorFlowBenchmark(args=benchmark_args)
    try:
        benchmark_args = parser.parse_args_into_dataclasses()[0]
    except ValueError as e:
        arg_error_msg = "Arg --no_{0} is no longer used, please use --no-{0} instead."
        begin_error_msg = " ".join(str(e).split(" ")[:-1])
        full_error_msg = ""
        depreciated_args = eval(str(e).split(" ")[-1])
        wrong_args = []
        for arg in depreciated_args:
            # arg[2:] removes '--'
            if arg[2:] in TensorFlowBenchmark.deprecated_args:
                # arg[5:] removes '--no_'
                full_error_msg += arg_error_msg.format(arg[5:])
            else:
                wrong_args.append(arg)
        if len(wrong_args) > 0:
            full_error_msg = full_error_msg + begin_error_msg + str(wrong_args)
        raise ValueError(full_error_msg)


SUMMARY:
**Function Summary**

*   **Purpose**: The main function `main()` parses command-line arguments, creates a TensorFlowBenchmark instance, and runs the benchmark.
*   **Functionality**: It utilizes an `HfArgumentParser` to parse arguments into dataclasses, handles deprecated argument warnings, and executes the benchmark using the `TensorFlowBenchmark` class.

**Key Implementation Details**

*   Uses an `HfArgumentParser` with `TensorFlowBenchmarkArguments` to parse command-line arguments.
*   Handles deprecated arguments raised by Python's `eval()` function on a string of deprecated options.
*   Includes error handling for invalid argument values, raising a `ValueError` with a formatted error message.

**Notable Dependencies and Requirements**

*   `HfArgumentParser`: A class from the Hugging Face Transformers library used to parse command-line arguments into dataclasses.
*   `TensorFlowBenchmarkArguments`: A custom class or interface extended from `arguments.BaseArguments`.
*   `TensorFlowBenchmark`: The benchmarking class executed after parsing arguments.
*   `python's eval() function`: Used for parsing deprecated argument names.

**Notes**: This code assumes the availability of specific classes and interfaces (`HfArgumentParser`, `TensorFlowBenchmarkArguments`, `TensorflowBenchmark`) in the project, which may require external libraries or custom definitions.


key: chroma:document
string_value:
CODE:

def list_field(default=None, metadata=None):


SUMMARY:
**Summary**

Concats `field` function with factory method creating an instance of the same type as `default`, injecting `metadata`. This function acts as a shortcut to create and configure custom fields.

**Key Implementation Details:**

- Utilizes `field` function provided by `django-felix` library.
- Employs `factory_method` to dynamically create an instance based on the `default` value.
- Passes `metadata` directly to the parent `field()` method for configuration.

**Notable Dependencies/Requirements:**

- Requires `django-felix` library, referencing `Field`.


key: chroma:document
string_value:
CODE:

def can_convert_to_int(string):
    try:
        int(string)
        return True
    except ValueError:


SUMMARY:
**Summary:**

The `can_convert_to_int` function takes a string as input and attempts to convert it to an integer using the built-in `int()` function. If successful, it returns `True`. Otherwise, it catches the `ValueError` exception and returns `False`.

* Main purpose: Determine if a given string can be converted to an integer.
* Key implementation detail: Conversion with try-except block for handling potential `ValueError`.
* Notable dependency/interface: Built-in Python data type `int()`.

**RAG Code Summary:** 

`can_convert_to_int(str string) : bool`

`def can_convert_to_int(string):`
  `try: int(string)`
`except ValueError: return False`
`return True`


key: chroma:document
string_value:
CODE:

def can_convert_to_float(string):
    try:
        float(string)
        return True
    except ValueError:


SUMMARY:
**Summary:**

```markdown
**Function:** `can_convert_to_float`
**Purpose:** Checks if a given string can be converted to a Float.

* Attempts to convert the input string to a float using `float()`.
* Returns True (representing successful conversion) or False on failure.

## Key Implementation Details

* Uses `try-except` block to handle `ValueError` exceptions raised during floating-point conversion.

## Notable Dependencies/Requirements**

* None. This function does not import any external modules or classes.
```

**RAG (Required Action Guide) Tags:**

* Primary Purpose: Convert input string to float
* Key Implementation Details: Floating-point conversion attempt and `try-except` block handling
* Notable Dependencies: None


key: chroma:document
string_value:
CODE:

class Plot:
    def __init__(self, args):
        self.args = args
        self.result_dict = defaultdict(lambda: {"bsz": [], "seq_len": [], "result": {}})

        with open(self.args.csv_file, newline="") as csv_file:
            reader = csv.DictReader(csv_file)
            for row in reader:
                model_name = row["model"]
                self.result_dict[model_name]["bsz"].append(int(row["batch_size"]))
                self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
                if can_convert_to_int(row["result"]):
                    # value is not None
                    self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
                        int(row["result"])
                    )
                elif can_convert_to_float(row["result"]):
                    # value is not None
                    self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
                        float(row["result"])
                    )

    def plot(self):
        fig, ax = plt.subplots()
        title_str = "Time usage" if self.args.is_time else "Memory usage"
        title_str = title_str + " for training" if self.args.is_train else title_str + " for inference"

        if not self.args.no_log_scale:
            # set logarithm scales
            ax.set_xscale("log")
            ax.set_yscale("log")

        for axis in [ax.xaxis, ax.yaxis]:
            axis.set_major_formatter(ScalarFormatter())

        for model_name_idx, model_name in enumerate(self.result_dict.keys()):
            batch_sizes = sorted(set(self.result_dict[model_name]["bsz"]))
            sequence_lengths = sorted(set(self.result_dict[model_name]["seq_len"]))
            results = self.result_dict[model_name]["result"]

            (x_axis_array, inner_loop_array) = (
                (batch_sizes, sequence_lengths) if self.args.plot_along_batch else (sequence_lengths, batch_sizes)
            )

            label_model_name = (
                model_name if self.args.short_model_names is None else self.args.short_model_names[model_name_idx]
            )

            for inner_loop_value in inner_loop_array:
                if self.args.plot_along_batch:
                    y_axis_array = np.asarray(
                        [results[(x, inner_loop_value)] for x in x_axis_array if (x, inner_loop_value) in results],
                        dtype=int,
                    )
                else:
                    y_axis_array = np.asarray(
                        [results[(inner_loop_value, x)] for x in x_axis_array if (inner_loop_value, x) in results],
                        dtype=np.float32,
                    )

                (x_axis_label, inner_loop_label) = (
                    ("batch_size", "len") if self.args.plot_along_batch else ("in #tokens", "bsz")
                )

                x_axis_array = np.asarray(x_axis_array, int)[: len(y_axis_array)]
                plt.scatter(
                    x_axis_array, y_axis_array, label=f"{label_model_name} - {inner_loop_label}: {inner_loop_value}"
                )
                plt.plot(x_axis_array, y_axis_array, "--")

            title_str += f" {label_model_name} vs."

        title_str = title_str[:-4]
        y_axis_label = "Time in s" if self.args.is_time else "Memory in MB"

        # plot
        plt.title(title_str)
        plt.xlabel(x_axis_label)
        plt.ylabel(y_axis_label)
        plt.legend()

        if self.args.figure_png_file is not None:
            plt.savefig(self.args.figure_png_file)
        else:


SUMMARY:
**Summary:**

The `Plot` class is designed to visualize training and inference time(memory) usage patterns of different models. It takes a CSV file containing model information as input.

**Main Purpose and Functionality:**

1. Load the specified CSV file, which contains model names and their corresponding training/inference statistics (e.g., batch size, sequence length, result values).
2. Plot the result values for each model in a scatter plot, with x-axis representing one of the statistics (batch size or sequence length) and y-axis representing the other statistic and its result value.

**Key Implementation Details:**

1. The `plot` method creates two subplots if plotting along batch vs sequence length, and otherwise plots results as functions of one statistic.
2. The scatter plot is generated using the seaborn `scatter` function, with line plots connecting each point for consistency.
3. The plot includes labels, titles, and legends to explain the data points.

**Notable Dependencies and Requirements:**

1. The class uses the `defaultdict` from the `collections` module to efficiently group results by model name.
2. It relies on the pandas library through CSV file reading (`csv.DictReader`).
3. The plot generation utilizes the matplotlib library.
4. Optional dependencies include seaborn for the scatter plot function and the plt.show() function for displaying plots, as well as PIL for saving PNG files.

**Interfaces Implemented:**

1. The `Plot` class adheres to the standard plotting APIs in matplotlib (e.g., `fig`, `ax`, `plt.scatter`, etc.) and supports additional options via the matplotlib rcParams (e.g., `'no_log_scale'`)


key: chroma:document
string_value:
CODE:

def main():
    parser = HfArgumentParser(PlotArguments)
    plot_args = parser.parse_args_into_dataclasses()[0]
    plot = Plot(args=plot_args)


SUMMARY:
**Summary:**

`main()` function initializes and executes a plot generation pipeline. It uses an `HfArgumentParser` to parse command-line arguments, creates a `Plot` object with the parsed data, and calls its `plot()` method.

**Key Details:**

*   Main entry point for the application
*   Uses `HfArgumentParser` to parse command-line arguments of type `PlotArguments`
*   Creates and initializes a `Plot` object (`plot`) with a reference to the parsed argument data.
*   Calls the `plot()` method on the `Plot` object.

**Notable Dependencies/Requirements:**

*   `HfArgumentParser`: A class for parsing command-line arguments
*   `PlotArguments` : The type of arguments to be parsed by `HfArgumentParser`
*   `Plot`: A class used for generating plots, with a method `plot()` that implements the actual plotting functionality.


key: chroma:document
string_value:
CODE:

def find_backend(line: str) -> Optional[str]:
    """
    Find one (or multiple) backend in a code line of the init.

    Args:
        line (`str`): A code line in an init file.

    Returns:
        Optional[`str`]: If one (or several) backend is found, returns it. In the case of multiple backends (the line
        contains `if is_xxx_available() and `is_yyy_available()`) returns all backends joined on `_and_` (so
        `xxx_and_yyy` for instance).
    """
    if _re_test_backend.search(line) is None:
        return None
    backends = [b[0] for b in _re_backend.findall(line)]
    backends.sort()


SUMMARY:
**Summary**

This function, `find_backend`, extracts and aggregates backend names (`b[0]`) from a given code line. It uses regular expressions to search for matching patterns and then returns the found backends in sorted order, optionally joined with `_and_`.

**Key Implementation Details**

* Uses `_re_test_backend` and `_re_backend` regular expression objects to search and find backend names.
* Returns `None` if no match is found.

**Dependencies**

* `_re_test_backend`, `_re_backend`
* `Optional[str]`

**Return Value**

* A string containing one or more backend names, separated by `_and_`. 

**Primary Functionality**
Find backend names from a given code line in an init file.


key: chroma:document
string_value:
CODE:

def read_init() -> Dict[str, List[str]]:
    """
    Read the init and extract backend-specific objects.

    Returns:
        Dict[str, List[str]]: A dictionary mapping backend name to the list of object names requiring that backend.
    """
    with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()

    # Get to the point we do the actual imports for type checking
    line_index = 0
    while not lines[line_index].startswith("if TYPE_CHECKING"):
        line_index += 1

    backend_specific_objects = {}
    # Go through the end of the file
    while line_index < len(lines):
        # If the line is an if is_backend_available, we grab all objects associated.
        backend = find_backend(lines[line_index])
        if backend is not None:
            while not lines[line_index].startswith("    else:"):
                line_index += 1
            line_index += 1

            objects = []
            # Until we unindent, add backend objects to the list
            while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8):
                line = lines[line_index]
                single_line_import_search = _re_single_line_import.search(line)
                if single_line_import_search is not None:
                    # Single-line imports
                    objects.extend(single_line_import_search.groups()[0].split(", "))
                elif line.startswith(" " * 12):
                    # Multiple-line imports (with 3 indent level)
                    objects.append(line[12:-2])
                line_index += 1

            backend_specific_objects[backend] = objects
        else:
            line_index += 1



SUMMARY:
**Function Summary: `read_init`**

* **Purpose and Functionality**: Extracts backend-specific objects from the `__init__.py` file of a Hugging Face transformer model, mapping each backend to its associated object names.
* **Key Implementation Details**:
	+ Reads and parses the entire `__init__.py` file.
	+ Searches for `if is_backend_available` statements to identify backend-specific imports.
	+ Parses multiple lines using regular expressions to extract object names from import statements.
* **Dependencies and Requirements**:
	+ Hugging Face transformer models (specifically, `PATH_TO_TRANSFORMERS`)
	+ `_re_single_line_import` regex pattern (internal function)
	+ `find_backend` function (imported but not explicitly shown in this code snippet)
* **Interfaces/Inherited:** This function inherits from no specific interface.

**Notes:**

* The function returns a dictionary with the format `{backend_name: [object_names]}`.
* The `_re_single_line_import` regex pattern is used to extract object names from single-line import statements.


key: chroma:document
string_value:
CODE:

def create_dummy_object(name: str, backend_name: str) -> str:
    """
    Create the code for a dummy object.

    Args:
        name (`str`): The name of the object.
        backend_name (`str`): The name of the backend required for that object.

    Returns:
        `str`: The code of the dummy object.
    """
    if name.isupper():
        return DUMMY_CONSTANT.format(name)
    elif name.islower():
        return DUMMY_FUNCTION.format(name, backend_name)
    else:


SUMMARY:
**Function Summary**

`create_dummy_object`: Returns code for a dummy object of specified type based on input `name`. The type is determined by the casing of `name`.

* **Purpose**: Generate code for a dummy object with given name and backend requirement.
* **Implementation Details**:
	+ Uses format strings (`DUMMY_CONSTANT`, `DUMMY_FUNCTION`, `DUMMY_CLASS`) to generate code.
	+ Includes `backend_name` in generated code when `name` is not all upper case or lower case.
* **Notable Dependencies**: Uses string formatting (`format`) and likely inherited from a parent class/inheritance hierarchy containing `DUMMY_CONSTANT`, `DUMMY_FUNCTION`, and `DUMMY_CLASS`. 

**Key Interfaces/Methods:**

- `DUMMY_CONSTANT`
- `DUMMY FUNCTION`
- `DUMMY CLASS`

**External Dependencies:** None


key: chroma:document
string_value:
CODE:

def create_dummy_files(backend_specific_objects: Optional[Dict[str, List[str]]] = None) -> Dict[str, str]:
    """
    Create the content of the dummy files.

    Args:
        backend_specific_objects (`Dict[str, List[str]]`, *optional*):
            The mapping backend name to list of backend-specific objects. If not passed, will be obtained by calling
            `read_init()`.

    Returns:
        `Dict[str, str]`: A dictionary mapping backend name to code of the corresponding backend file.
    """
    if backend_specific_objects is None:
        backend_specific_objects = read_init()

    dummy_files = {}

    for backend, objects in backend_specific_objects.items():
        backend_name = "[" + ", ".join(f'"{b}"' for b in backend.split("_and_")) + "]"
        dummy_file = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n"
        dummy_file += "from ..utils import DummyObject, requires_backends\n\n"
        dummy_file += "\n".join([create_dummy_object(o, backend_name) for o in objects])
        dummy_files[backend] = dummy_file



SUMMARY:
**Function Summary: create_dummy_files**

**Purpose**: Creates the content of dummy files based on backend-specific objects.

**Functionality**

* Takes an optional dictionary `backend_specific_objects` mapping backend names to lists of backend-specific objects.
* If none provided, reads them from `read_init()`.
* Creates a dictionary `dummy_files` with backend names as keys and their corresponding code as values.
* For each backend:
	+ Constructs the code by concatenating object imports and calls to `create_dummy_object` method.
	+ Prefers formatted output for list of backend objects.

**Key Implementation Details**

* Uses `read_init()` to retrieve backend-specific objects when not provided.
* Employed string formatting to nicely display list of backend objects.

**Notable Dependencies**

* `read_init()`
* `create_dummy_object()
* `DummyObject` class from `..utils` module
* External dependencies: requires `make` command and directory structure


key: chroma:document
string_value:
CODE:

def check_dummies(overwrite: bool = False):
    """
    Check if the dummy files are up to date and maybe `overwrite` with the right content.

    Args:
        overwrite (`bool`, *optional*, default to `False`):
            Whether or not to overwrite the content of the dummy files. Will raise an error if they are not up to date
            when `overwrite=False`.
    """
    dummy_files = create_dummy_files()
    # For special correspondence backend name to shortcut as used in utils/dummy_xxx_objects.py
    short_names = {"torch": "pt"}

    # Locate actual dummy modules and read their content.
    path = os.path.join(PATH_TO_TRANSFORMERS, "utils")
    dummy_file_paths = {
        backend: os.path.join(path, f"dummy_{short_names.get(backend, backend)}_objects.py")
        for backend in dummy_files.keys()
    }

    actual_dummies = {}
    for backend, file_path in dummy_file_paths.items():
        if os.path.isfile(file_path):
            with open(file_path, "r", encoding="utf-8", newline="\n") as f:
                actual_dummies[backend] = f.read()
        else:
            actual_dummies[backend] = ""

    # Compare actual with what they should be.
    for backend in dummy_files.keys():
        if dummy_files[backend] != actual_dummies[backend]:
            if overwrite:
                print(
                    f"Updating transformers.utils.dummy_{short_names.get(backend, backend)}_objects.py as the main "
                    "__init__ has new objects."
                )
                with open(dummy_file_paths[backend], "w", encoding="utf-8", newline="\n") as f:
                    f.write(dummy_files[backend])
            else:
                raise ValueError(
                    "The main __init__ has objects that are not present in "
                    f"transformers.utils.dummy_{short_names.get(backend, backend)}_objects.py. Run `make fix-copies` "
                    "to fix this."


SUMMARY:
**Function Summary: check_dummies**

**Purpose:** Verify the integrity of dummy files and optionally update their content.

**Functionality:**

1. Checks if dummy file paths exist and reads their contents.
2. Compares actual contents with expected ones. If updated, updates the file.
3. Raises `ValueError` if dummy files are outdated (non-present) when `overwrite=False`.

**Key Implementation Details:**

* Uses `create_dummy_files()` to generate dictionary of dummy files.
* Uses a mapping (`short_names`) for shortcutting backend names.

**Notable Dependencies and Requirements:**

* `utils/dummy_xxx_objects.py` file
* `os` module (for interacting with the file system)
* `pathlib` (not explicitly mentioned, but implied by `PATH_TO_TRANSFORMERS`)
* `make` command (to run `fix-copies` to update outdated dummy files)


key: chroma:document
string_value:
CODE:

def get_module_source_from_name(module_name: str) -> str:
    # Extract the source code from the module name
    spec = importlib.util.find_spec(module_name)
    if spec is None or spec.origin is None:
        raise ValueError(f"Cannot open file associated with {module_name} module.")

    with open(spec.origin, "r", encoding="utf-8") as file:
        source_code = file.read()


SUMMARY:
**Summary**

`get_module_source_from_name`: Extracts(source code) from a given Python module name (`module_name`). 

### Key Details

* Uses `importlib.util.find_spec` to locate the module spec.
* Returns the read contents of the module's associated file.

### Implementation Notes

* Checks for existence and validity of the module spec before proceeding.
* Raises a `ValueError` if the file associated with the module is not found or cannot be opened.
* reads the source code from the file using UTF-8 encoding.

### Dependencies and Interfaces

* `importlib.util`: Module to locate Python modules.
* `python files/utf-8 encoding (`encoding="utf-8"`): Specifies character encoding for reading the file.


key: chroma:document
string_value:
CODE:

def preserve_case_replace(text, patterns: dict, default_name: str):
    # Create a regex pattern to match all variations
    regex_pattern = "|".join(re.escape(key) for key in patterns.keys())
    compiled_regex = re.compile(f"(?<![a-z0-9])({regex_pattern})(.|$)", re.IGNORECASE | re.DOTALL)

    def replace(match):
        matched_pattern = match.group(1)
        next_char = match.group(2)
        new_pattern = patterns.get(matched_pattern, default_name)

        # In this case, the cased old model did not respect CamelCase and was all UPPERCASE, so we need to rely on next char
        # The heuristic is: if next char is not a letter, then it is not part of a model name and result should be `new_name`.upper()
        if len(patterns) == 2 and matched_pattern.isupper():
            if not next_char.isalpha():
                # `new_name.upper()` is just the other entry for `matched_pattern.lower()`, uppercased
                new_pattern = patterns[matched_pattern.lower()].upper()

        return new_pattern + next_char



SUMMARY:
**Function Summary: preserve_case_replace**

Preserves sentence case in a given `text` by replacing matched patterns with corresponding names from the `patterns` dictionary. The function uses a regex pattern to match all variations of model names.

**Key Functionality:**

*   Preserves original text while replacing matched patterns.
*   Replaces patterns based on the order and key-value pairs in `patterns`.
*   Uses case-insensitive matching for non-alphanumeric characters.

**Important Implementation Details:**

*   Regex pattern `(?<![a-z0-9])({regex_pattern})(.|$)` matches all variations of model names while preserving sentence case.
*   The `replace` function uses the matched pattern, next character, and corresponding name from `patterns` to replace the original text.

**Notable Dependencies:**

*   `re` module for regex operations.
*   Python's built-in string methods (e.g., `isalpha()`, `upper()`).

**External Dependencies:**

*   None specified in the provided code chunk. However, it can be assumed that any library or framework being used by the target project may also require this function to function appropriately.


key: chroma:document
string_value:
CODE:

def get_cased_name(lowercase_name: str) -> str:
    """From a model name in lowercase in the format `my_model`, return the cased name in the format `MyModel`."""
    if lowercase_name in CONFIG_MAPPING_NAMES:
        return CONFIG_MAPPING_NAMES[lowercase_name].replace("Config", "")
    else:


SUMMARY:
**Summary:**

This function, `get_cased_name`, takes a lowercase model name as input and returns the cased version. If the input matches a predefined mapping, it replaces "Config" with an empty string; otherwise, it applies title case to the split and joined components.

**Key Implementation Details:**

* Replaces "Config" for matched mappings
* Applies title case using `str.title()` for unmatched inputs

**Dependencies/Interfaces:**

* `CONFIG_MAPPING_NAMES`: a dictionary-like data structure (not defined in this code snippet)

**Notable Method Signatures:**

* `get_cased_name(lowercase_name: str) -> str`


key: chroma:document
string_value:
CODE:

def get_lowercase_name(cased_name: str) -> str:
    """From a model name in Camelcase in the format `MyModel`, return the lowercase name in the format `my_model`."""
    inverse_mapping = {value: key for key, value in CONFIG_MAPPING_NAMES.items()}
    if cased_name + "Config" in inverse_mapping:
        return inverse_mapping[cased_name + "Config"]
    else:


SUMMARY:
**Function Summary:**

`get_lowercase_name`

* **Purpose:** Convert a Camelcase name to lowercase and format according to the ` CONFIG_MAPPING_NAMES`.
* **Functionality:** 
	+ Checks if a Config variant exists in the inverse mapping for the given Camelcase name.
	+ If not, returns the lowered Camelcase name by splitting into words and joining with underscores.

**Key Implementation Details:**

* Uses regular expressions (`re.findall`) to extract words from the Camelcase input string.
* Applies `str.lower()` to each word before rejoining them with underscores.

**Notable Dependencies or Requirements:**

* `CONFIG_MAPPING_NAMES` (injected as a constant)
* `re` (regular expression module)


key: chroma:document
string_value:
CODE:

class ReplaceNameTransformer(m.MatcherDecoratableTransformer):
    """A transformer that replaces `old_name` with `new_name` in comments, string and any references.
    It should take into account name like `MyNewModel`, or `my_new_model`. Without using the AUTO_MAPPING.
    Supported renaming patterns:
        - llama -> my_new_model     and     my_new_model    -> llama
        - Llama -> MyNewModel       and     MyNewModel      -> Llama
        - LLAMA -> MY_NEW_MODEL     and     MY_NEW_MODEL    -> LLAMA
        - LLaMa -> MyNewModel       abd     MyNewModel      -> Llama
    """

    def __init__(self, old_name: str, new_name: str, original_new_model_name: str = "", only_doc: bool = False):
        super().__init__()
        self.old_name = old_name
        self.new_name = new_name
        self.cased_new_name = get_cased_name(self.new_name)
        self.cased_old_name = get_cased_name(self.old_name)
        self.patterns = {
            old_name: new_name,
            old_name.upper(): new_name.upper(),
            # For some old models, `self.cased_old_name` == `old_name.upper()` in which case this overwrite previous entry
            self.cased_old_name: self.cased_new_name,
        }
        # In case new_name is a prefix alias, and not the original new model name
        self.original_new_model_name = original_new_model_name
        self.only_doc = only_doc

    def _replace_name(self, original_node, updated_node):
        if re.findall(r"# Copied from", updated_node.value):
            return cst.RemoveFromParent()
        update = preserve_case_replace(updated_node.value, self.patterns, self.cased_new_name)
        return updated_node.with_changes(value=update)

    @m.leave(m.SimpleString() | m.Comment())
    def replace_name(self, original_node, updated_node):
        return self._replace_name(original_node, updated_node)

    def leave_Name(self, original_node, updated_node):
        if not self.only_doc:
            return self._replace_name(original_node, updated_node)
        return updated_node

    def leave_ImportFrom(self, original_node, updated_node):
        """The imports from other file types (configuration, processing etc) should use original model name."""
        if self.original_new_model_name != self.new_name and m.matches(updated_node.module, m.Name()):
            patterns = "|".join(ALL_FILE_TYPES)
            regex = rf"({patterns})_{self.new_name}"
            new_source = re.sub(
                regex, lambda m: f"{m.group(1)}_{self.original_new_model_name}", updated_node.module.value
            )
            updated_node = updated_node.with_changes(module=updated_node.module.with_changes(value=new_source))


SUMMARY:
**Summary**

The `ReplaceNameTransformer` class is designed to replace specific names in comments, strings, and references. It handles various naming patterns, considering case sensitivity and prefixes/qualifiers.

**Key Functionality:**

1.  Replaces `old_name` with `new_name` in:
    *   Comments
    *   Strings
    *   References

2.  Supports different renaming patterns:

    *   `llama -> my_new_model`
    *   `Llama -> MyNewModel`
    *   `LLAMA -> MY_NEW_MODEL`
    *   `(prefix)_LLaMa_(suffix)`

**Implementation Details:**

*   `_replace_name` method applies the replacement rules to an updated node.

*   The transformer considers the case of the original name and uses a `patterns` dictionary for replacements.
*   It preserves the original meaning by allowing certain suffixes (like `.py`) to remain unchanged.

**Dependencies/Notable Requirements:**

*   Uses Python's `re` module for regular expressions
*   Leverages the `cst` library for parsing nodes

Here's a concise summary of key classes and functions:

*   `class ReplaceNameTransformer(m.MatcherDecoratableTransformer):`

    *   Initializes with `old_name`, `new_name`, optional `original_new_model_name`, and `only_doc`.
    *   Provides `replace_name`, `leave_Name` and `leave_ImportFrom` methods to apply the transformer transformations

The full implementation showcases how this transformer can be used to standardize naming conventions within a larger system.


key: chroma:document
string_value:
CODE:

def SUPER_CALL_NODE(func_name):


SUMMARY:
**Summary:**

The `SUPER_CALL_NODE` function generates an abstract syntax tree (AST) node for the `super()` call in Python.

**Key Implementation Details:**

* The `m.Call` node represents the call operation with its attributes.
* The `attr` attribute within the `Call` node specifies the `func_name` parameter, which is passed from the caller function.
* A nested `m.Call` node represents the built-in `super()` function and has a specific structure, calling another inner `m.Name` node named `"super"`.
* Both nodes rely on another higher-level node `m.Attribute`, indicating method calls.

**Notable Dependencies/Requirements:**

* The function uses the `ast` module (implied but not shown) to create an abstract syntax tree.
* It relies on a meta-programming system, possibly a Python parser generator like Parsec or PyParsing.


key: chroma:document
string_value:
CODE:

def is_call_to_super(node, func_name):
    return m.matches(
        node, m.SimpleStatementLine(body=[m.Return(SUPER_CALL_NODE(func_name)) | m.Expr(SUPER_CALL_NODE(func_name))])


SUMMARY:
**Summary:**

`is_call_to_super` function checks if the given `node` represents a call to the superclass (__Super__) from a method named `func_name`. 

**Key Implementation Details:**
- Uses regular expression (`m.matches`) to match a specific pattern in the node.
- Expects a match with either a return statement calling `SUPER_CALL_NODE(func_name)` or an expression also calling `SUPER_CALL_NODE(func_name)`.

**Notable Dependencies/Requirements:**

- `SuperCallNode` is assumed to be a previously defined class or function.
- External dependency on regular expression engine (`m`) using `m.matches`. 

Note: Without additional context, the exact nature of `SUPER_CALL_NODE(func_name)`, `m`, and related classes/functions cannot be precisely determined.


key: chroma:document
string_value:
CODE:

def get_full_attribute_name(node: cst.Attribute | cst.Name) -> str | None:
    """Get the full name of an Attribute or Name node (e.g. `"nn.Module"` for an Attribute representing it). If the
    successive value of an Attribute are not Name nodes, return `None`."""
    if m.matches(node, m.Name()):
        return node.value
    elif m.matches(node, m.Attribute()):
        if not m.matches(node.attr, m.Name()):
            return None
        name = node.attr.value
        new_node = node.value
        while m.matches(new_node, m.Attribute()):
            if not m.matches(new_node.attr, m.Name()):
                return None
            name = new_node.attr.value + "." + name
            new_node = new_node.value
        if not m.matches(new_node, m.Name()):
            return None
        return new_node.value + "." + name


SUMMARY:
**Summary**

`get_full_attribute_name` function extracts the full attribute name from an AST node (either `Attribute` or `Name`). It traverses attributes and checks if successive values are `Name` nodes. If successful, it returns the fully qualified name.

**Key Details**

* Function takes an `AST` node (`cst.Attribute | cst.Name`) as input
* Returns a string representing the full attribute name or `None` if incomplete
* Uses a regular expression (m) to match node types

**Implementation Choices**

* Uses recursive descent approach for traversing attributes
* Employed a try-and-contract approach to handle cases with incomplete successive values


key: docstring
string_value:
# Transformer class to replace ClassB.call_to_method and ClassB().call_to_method with super().call_to_method


key: chroma:document
string_value:
CODE:
# Transformer class to replace ClassB.call_to_method and ClassB().call_to_method with super().call_to_method
class ReplaceMethodCallTransformer(cst.CSTTransformer):
    def __init__(self, all_bases: Set[str]):
        self.all_bases = all_bases

    def leave_Attribute(self, original_node: cst.Attribute, updated_node: cst.Attribute) -> cst.CSTNode:
        # Handle ClassB.call_to_method or module.classB.call_to_method
        if (
            m.matches(original_node.value, m.Name() | m.Attribute())
            and get_full_attribute_name(original_node.value) in self.all_bases
            and m.matches(original_node.attr, m.Name())
        ):
            # Replace with super().call_to_method
            return updated_node.with_changes(
                value=cst.Call(cst.Name("super")),
            )
        # Handle ClassB().call_to_method or module.ClassB().call_to_method
        elif (
            m.matches(original_node.value, m.Call())
            and m.matches(original_node.value.func, m.Name() | m.Attribute())
            and get_full_attribute_name(original_node.value.func) in self.all_bases
            and m.matches(original_node.attr, m.Name())
        ):
            # Replace with super().call_to_method
            return updated_node.with_changes(value=cst.Call(cst.Name("super")))
        return updated_node

    def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.CSTNode:
        # Check if the function being called is of the form ClassB().func_a or ClassB.func_a
        if m.matches(original_node.func, m.Attribute()) and (
            # Match ClassB().func_a(...) or module
            (
                m.matches(original_node.func.value, m.Call())
                and m.matches(original_node.func.value.func, m.Name() | m.Attribute())
                and get_full_attribute_name(original_node.func.value.func) in self.all_bases
            )
            or
            # Match ClassB.func_a(...)
            (
                m.matches(original_node.func.value, m.Name() | m.Attribute())
                and get_full_attribute_name(original_node.func.value) in self.all_bases
            )
        ):
            # Check if the first argument is 'self', and remove it
            if len(original_node.args) > 0 and m.matches(original_node.args[0].value, m.Name("self")):
                # Create the new argument list without 'self'
                new_args = updated_node.args[1:]
            else:
                new_args = updated_node.args

            return updated_node.with_changes(args=new_args)


SUMMARY:
**Class Summary**

`ReplaceMethodCallTransformer` is a CST transformer that replaces `ClassB.call_to_method` and `ClassB().call_to_method` with `super().call_to_method`.

**Primary Purpose**

The main purpose of this class is to transform Python code by replacing method calls in the specified formats with the equivalent Super class call.

**Key Implementation Details**

1. **leave_Attribute**: This function handles case where attribute name is present as a string (`ClassB.call_to_method`). It checks if the attribute name is in `all_bases` and replaces it with `super().call_to_method`.
2. **leave_Call**: This function handles cases where the method call contains an instance of `(module)` or only the class method call (e.g. `ClassB.func_a`). It checks for the presence of 'self' as an argument, removes it if present and updates the arguments of the function call.

**Notable Dependencies/Requirements**

* `cst` module for working with Python abstract syntax trees (ASTs)
* `Set[str]` for storing class names
* `get_full_attribute_name` and `m.matches` functions appear to be custom utility functions not explicitly included in the provided code.


key: chroma:document
string_value:
CODE:

def get_docstring_indent(docstring):
    # Match the first line after the opening triple quotes
    match = re.search(r'(?:"""|\'\'\'|```)\n(\s+)', docstring)
    if match:
        # Return the indentation spaces captured
        return len(match.group(1))


SUMMARY:
**Function Summary**

```
get_docstring_indent function
=============================

Returns the number of indentation spaces preceding the first line after triple quotes in a docstring.

### Purpose

Extracts and returns the indentation level from the matched opening docstring triplet.

### Implementation Details

- Uses regular expression to match the opening triplet and capture the following whitespace content.
- Returns the length of the captured whitespace as the indentation distance.

### Dependency

None
```


key: chroma:document
string_value:
CODE:

def merge_docstrings(original_docstring, updated_docstring):
    # indent_level = get_docstring_indent(updated_docstring)
    original_level = get_docstring_indent(original_docstring)
    if not re.findall(r"\n\s*Args:\n", updated_docstring):
        # Split the docstring at the example section, assuming `"""` is used to define the docstring
        parts = original_docstring.split("```")
        if "```" in updated_docstring and len(parts) > 1:
            updated_docstring = updated_docstring.lstrip('r"')
            new_parts = updated_docstring.split("```")
            if len(new_parts) != 3:
                raise ValueError("There should only be one example, and it should have opening and closing '```'")
            parts[1] = new_parts[1]
            updated_docstring = "".join(
                [
                    parts[0].rstrip(" \n") + new_parts[0],
                    f"\n{original_level*' '}```",
                    parts[1],
                    "```",
                    parts[2],
                ]
            )
        elif updated_docstring not in original_docstring:
            # add tabulation if we are at the lowest level.
            if re.search(r"\n\s*.*\(.*\)\:\n\s*\w", updated_docstring):
                updated_docstring = updated_docstring.replace("\n    ", "\n        ")
            updated_docstring = original_docstring.rstrip('"') + "\n" + updated_docstring.lstrip('r"\n')


SUMMARY:
**Summary**

The `merge_docstrings` function merges two docstrings, aiming to blend the style of both into a single cohesive doc string. Specifically:

### Main Purpose and Functionality

* Merges two doc strings (`original_docstring` and `updated_docstring`) into one.
* Preserves indentation structure from `original_docstring`.
* Updates `Args:` section with new content from `updated_docstring`.

### Key Implementation Details

* Uses regular expression to detect and replace `Args:` format in `updated_docstring`.
* Corrects inconsistent indentation by tabulation.
* Handles different cases of doc string formatting, including single-step and multi-step examples.

### Notable Dependencies or Requirements

* Utilizes string manipulation functions (e.g., `rstrip`, `lstrip`, `split`) and regular expressions (e.g., `re.findall`, `re.search`).
* Assumes the use of triple quoting (`"""`) for defining docstrings.
* Expected input: Two doc strings, with an optional third example section using triple quotes.


key: chroma:document
string_value:
CODE:

class SuperTransformer(cst.CSTTransformer):
    METADATA_DEPENDENCIES = (ParentNodeProvider,)

    def __init__(self, python_module: cst.Module, original_methods, updated_methods, all_bases=None):
        self.python_module = python_module
        self.original_methods = original_methods
        self.updated_methods = updated_methods
        self.all_assign_target = {}
        self.deleted_targets = {}  # child node can delete some arguments
        self.all_bases = all_bases or []
        self.transformer = ReplaceMethodCallTransformer(set(self.all_bases))

    def update_body(self, existing_body, new_statements):
        """
        Helper method to update the body by removing duplicates before adding new statements.
        `existing_body` is the body of the original method, the parent class
        `new_statements` are the additional statements
        """
        deduplicated_new_body = []
        existing_nodes = set()
        for node in new_statements:
            if m.matches(node, m.SimpleStatementLine(body=[m.Assign()])):
                target = self.python_module.code_for_node(node.body[0].targets[0].target)
                self.all_assign_target[target] = node
            if m.matches(node, m.SimpleStatementLine(body=[m.Del()])):
                target = self.python_module.code_for_node(node.body[0].target)
                self.deleted_targets[target] = node

        for stmt in existing_body:
            if m.matches(stmt, m.SimpleStatementLine(body=[m.Assign()])):
                target = self.python_module.code_for_node(stmt.body[0].targets[0].target)
                if target in self.deleted_targets:
                    continue
                if target in self.all_assign_target:
                    stmt = self.all_assign_target[target]
            # Skip the docstring (will be added later on, at the beginning)
            elif m.matches(stmt, DOCSTRING_NODE):
                continue
            comment_less_code = re.sub(r"#.*", "", self.python_module.code_for_node(stmt)).strip()
            comment_less_code = re.sub(r"\ *\n", "\n", comment_less_code).strip()
            deduplicated_new_body.append(stmt)
            existing_nodes.add(comment_less_code)

        for node in new_statements:
            code = self.python_module.code_for_node(node)
            comment_less_code = re.sub(r"#.*", "", code).strip()
            comment_less_code = re.sub(r"\ *\n", "\n", comment_less_code).strip()
            if node not in deduplicated_new_body and comment_less_code not in existing_nodes:
                if not m.matches(node, m.SimpleStatementLine(body=[m.Del()])):
                    deduplicated_new_body.append(node)
                    existing_nodes.add(comment_less_code)

        deduplicated_new_body = self._fix_post_init_location(deduplicated_new_body)

        return deduplicated_new_body

    def _fix_post_init_location(self, new_body: list[cst.CSTNode]):
        """Fix the location of the `post_init()` in the new body, if we added statements after the call to
        `super()` (it needs to be the very last statement called)"""
        # Fix the post_init() that has to be last
        for i, node in enumerate(new_body):
            code = self.python_module.code_for_node(node)
            comment_less_code = re.sub(r"#.*", "", code).strip()
            comment_less_code = re.sub(r"\ *\n", "\n", comment_less_code).strip()
            if "self.post_init(" in comment_less_code and i < len(new_body) - 1:
                # Remove it and add it again at the end
                new_body.pop(i)
                new_body.append(node)
                break
        return new_body

    def _fix_init_location(self, new_body):
        """Fix the location of the `super().__init__()` in the new body, if we had new statements before it."""
        start_index = 0
        for i, node in enumerate(new_body):
            if m.matches(node, DOCSTRING_NODE) and i == start_index:
                start_index += 1
                continue
            code = self.python_module.code_for_node(node)
            comment_less_code = re.sub(r"#.*", "", code).strip()
            comment_less_code = re.sub(r"\ *\n", "\n", comment_less_code).strip()
            if "super().__init__" in comment_less_code and i > start_index:
                # Remove it and add it again at the top after the docstrings
                node = new_body.pop(i)
                new_body = new_body[:start_index] + [node] + new_body[start_index:]
                break
        return new_body

    def replace_super_calls(self, node: cst.IndentedBlock, func_name: str) -> cst.CSTNode:
        """Updates the body of the input `node`'s `func_name` function by replacing calls
        to super().func_name() with the source code of the parent class' `func_name`.
        It keeps everything that is defined before `super().func_name()`.
        """
        self.has_docstring = False
        parent_has_docstring = False
        if func_name in self.original_methods:
            parent_has_docstring = m.matches(self.original_methods[func_name].body.body[0], DOCSTRING_NODE)
        new_body = []
        has_super_call = False

        for i, expr in enumerate(node.body):
            if is_call_to_super(expr, func_name):
                has_super_call = True
                new_body.extend(self.update_body(self.original_methods[func_name].body.body, node.body[i + 1 :]))
                new_body = self._fix_init_location(new_body)
            else:
                expr = expr.visit(self.transformer)
            if m.matches(expr, DOCSTRING_NODE):
                self.has_docstring = True
                if parent_has_docstring:  # actually here we ought to de-duplicate?
                    original_docstring = self.original_methods[func_name].body.body[0].body[0].value.value
                    updated_docstring = expr.body[0].value.value
                    merged_doc = merge_docstrings(original_docstring, updated_docstring)
                    new_node = [expr.with_changes(body=[cst.Expr(value=cst.SimpleString(value=merged_doc))])]
                else:
                    new_node = [expr]
                new_body.extend(new_node)
            elif not m.matches(expr, m.SimpleStatementLine(body=[m.Del()])) and not has_super_call:
                new_body.append(expr)
        if not self.has_docstring and parent_has_docstring:
            new_body = [self.original_methods[func_name].body.body[0]] + new_body
        return node.with_changes(body=new_body)

    def leave_FunctionDef(self, original_node: cst.Call, updated_node: cst.Call) -> cst.CSTNode:
        if updated_node.name.value in self.updated_methods:
            name = updated_node.name.value
            new_body = self.replace_super_calls(updated_node.body, name)
            return updated_node.with_changes(body=new_body, params=updated_node.params)
        return updated_node

    def leave_Return(self, original_node: cst.Return, updated_node: cst.Return) -> cst.CSTNode:
        """ "When a return statement is reached, it is replaced with the unrolled super code"""
        if m.matches(updated_node.value, m.Call(func=m.Attribute(attr=m.Name("super")))):
            func_def = self.get_metadata(ParentNodeProvider, original_node)
            if m.matched(func_def, m.FunctionDef()) and func_def.name.value in self.original_methods:
                updated_return_value = updated_node.value.with_changes(
                    args=[
                        cst.Arg(
                            value=cst.Call(func=cst.Name("super"), args=[cst.Arg(value=cst.Name(func_def.name.value))])
                        )
                    ]
                )
                return updated_node.with_changes(value=updated_return_value)


SUMMARY:
**Summary:**

This class `SuperTransformer` is designed to transform Python code that references `super()` calls. It achieves this by:

*   **Replacing `super()` calls**: Replaces calls to `super()` with the source code of the parent class's method implementation.
*   **De-duplicating docstrings**: Merges overlapping docstrings from different sources in a unified way.

**Key Implementation Details:**

1.  **Main Functionality**: `replace_super_calls` function
    *   Updates the body of a function node by replacing calls to `super()` with the source code of the parent class's method implementation.
2.  **Docstring Handling**
    *   Uses separate functions `_fix_post_init_location` and `_fix_init_location` to handle docstrings in different locations (at the end, or at the top).
3.  **Visitor Pattern**: Utilizes a visitor pattern (`visit`) to transform AST nodes.

**Notable Dependencies:**

1.  `ParentNodeProvider`
2.  `m` (a module for parsing and manipulating code)
3.  `DOCSTRING_NODE` (a node representing docstrings in the AST)


key: chroma:document
string_value:
CODE:

def find_all_dependencies(
    dependency_mapping: Dict[str, set],
    start_entity: str | None = None,
    initial_dependencies: set | None = None,
    initial_checked_dependencies: set | None = None,
    return_parent: bool = False,
) -> list | set:
    """Return all the dependencies of the given `start_entity` or `initial_dependencies`. This is basically some kind of
    BFS traversal algorithm. It can either start from `start_entity`, or `initial_dependencies`.

    Args:
        dependency_mapping (`Dict[str, set]`):
            A mapping from entities (usually function/assignment names), to immediate dependencies. That is, for function names,
            a mapping {"foo": {"bar", "test"}} would indicate that functions `bar` and `test` are immediately called
            in `foo`'s definition.
        start_entity (str | None, *optional*):
            A key of `dependency_mapping`, indicating from which entity to start the search.
        initial_dependencies (set | None, *optional*):
            If `start_entity` is not provided, this can be used as an alternative. In this case, the search will continue
            from all the entities in `initial_dependencies`, if they are in `dependency_mapping`.
        initial_checked_dependencies (set | None, *optional*):
            If provided, entities already present in `initial_checked_dependencies` will not be part of the returned dependencies.
        return_parent (bool, *optional*):
            If `True`, will return a list consisting of tuples (dependency, parent) instead of a simple set of dependencies. Note
            that the order of the items in the list reflects the traversal order. Thus, no parent can ever appear before childs.
    Returns:
        A set of all the dependencies, or a list of tuples `(dependency, parent)` if `return_parent=True`.

    Example:
    Given the following structure in the `modular_xxx.py` file:
    ```
    def foo1():
        pass

    def foo2():
        pass

    def bar():
        foo1()

    def foobar():
        bar()
        foo2()

    class MyLayer(SomeOtherModelLayer):
        def forward(...):
            foobar()
    ```
    and the `dependency_mapping` created when visiting the `modular_xxx.py` file, we get:
    ```
    dependency_mapping = {'bar': {'foo1'}, 'foobar': {'bar', 'foo2'}}
    find_all_dependencies(dependency_mapping, start_entity='foobar', return_parent=True)
    >>> [('bar', 'foobar'), ('foo2', 'foobar'), ('foo1', 'bar')]
    ```
    That is, all the functions needed (and potentially their immediate parent) so that the function to be added
    in MyLayer (`foobar`) can work correctly.
    """
    if initial_dependencies is None and start_entity is not None:
        initial_dependencies = dependency_mapping[start_entity]
    if initial_checked_dependencies is None:
        initial_checked_dependencies = set()

    dependency_queue = deque(initial_dependencies)
    all_dependencies = set()
    all_dependencies_with_parent = []
    checked_dependencies = set(initial_checked_dependencies)
    parents = {initial_dep: start_entity for initial_dep in initial_dependencies}
    while len(dependency_queue) > 0:
        # Pick element to visit
        current = dependency_queue.popleft()
        if current not in checked_dependencies:
            # Add the dependencies
            all_dependencies.add(current)
            all_dependencies_with_parent += [(current, parents[current])]
            if current in dependency_mapping.keys():
                # Update dependency queue
                dependency_queue.extend(dependency_mapping[current])
                parents.update({dep: current for dep in dependency_mapping[current]})
            # add visited node to the list
            checked_dependencies.add(current)

    if not return_parent:
        return all_dependencies
    # no child can ever appear before its parent thanks to the queue (needed to add them at the correct location in the body later)


SUMMARY:
Here is a concise summary of the code:

**Function Purpose and Functionality**

This function performs a breadth-first traversal (BFS) on a dependency graph, where each entity represents a function or assignment name. It finds all dependencies for a given `start_entity` or a set of initial dependencies.

**Main Implementation Details**

* The function uses a queue to efficiently traverse the graph.
* It keeps track of visited entities and their dependencies using sets.
* If `return_parent=True`, it returns a list of tuples (dependency, parent) instead of just a set of dependencies.

**Key Method Signatures and Interfaces**

* `find_all_dependencies(Dict[str, set], str | None = None, ..., return_parent: bool = False -> list | set)`
* `deque()`
* `set()`

**Notable Dependencies or Requirements**

* `dependency_mapping`: a dictionary mapping entities to their dependencies.
* `start_entity`, `initial_dependencies`, and `return_parent` are optional parameters used to customize the traversal.

**Example Use Case**

Given a `modular_xxx.py` file with function definitions, this function returns all dependencies needed for a given entity or initial set of dependencies.


key: chroma:document
string_value:
CODE:

class ClassDependencyMapper(CSTVisitor):
    """A visitor which is designed to analyze a single class node to get all its dependencies that are shared with the set of
    `global_names`.
    """

    def __init__(
        self, class_name: str, global_names: set[str], objects_imported_from_modeling: set[str] | None = None
    ):
        super().__init__()
        self.class_name = class_name
        self.dependencies = set()
        self.global_names = global_names
        self.objects_imported_from_modeling = (
            set() if objects_imported_from_modeling is None else objects_imported_from_modeling
        )

    def visit_Name(self, node):
        if (
            node.value != self.class_name
            and node.value in self.global_names
            and node.value not in self.objects_imported_from_modeling
        ):


SUMMARY:
**ClassDependencyMapper Summary**

The `ClassDependencyMapper` class is a custom visitor for analyzing class nodes to identify shared dependencies with `global_names`. It primarily maps shared node values to set of dependencies.

**Key Implementation Details:**

* Visits each name node in the class definition.
* Checks if name value matches global names (excluding imported from modeling) and adds it to the dependencies set.

**Notable Dependencies or Requirements:**

* `global_names`: a set of global names used for dependency analysis.
* `objects_imported_from_modeling`: an optional set of objects imported from modeling, defaulting to an empty set if not provided.
* `_CSTVisitor`: the base visitor class inherited by this class.

Note that external dependencies are not explicitly mentioned in this code snippet.


key: chroma:document
string_value:
CODE:

def dependencies_for_class_node(node: cst.ClassDef, global_names: set[str]) -> set:
    """Create immediate dependencies for a class node based on the `global_names`."""
    temp_module = cst.Module(body=[node])
    visitor = ClassDependencyMapper(node.name.value, global_names)
    temp_module.visit(visitor)


SUMMARY:
**Summary:**

Creates immediate dependencies for a class node based on `global_names`.

**Main Purpose and Functionality:** 
The function `dependencies_for_class_node` takes a `ClassDef` node and a set of global names, and returns the dependencies required by that class.

**Key Implementation Details:**

- Creates a temporary module with the input class node
- Instantiates a `ClassDependencyMapper` visitor with the class name and global names
- Visits the temporary module, allowing the visitor to inspect dependencies

**Notable Dependencies or Requirements:**

- `cst` (Abstract Syntax Tree) module for parsing
- `ClassDependencyMapper` visitor for dependency resolution
- `global_names` set of imported names for dependency filtering


key: chroma:document
string_value:
CODE:

def augmented_dependencies_for_class_node(
    node: cst.ClassDef, mapper: "ModuleMapper", objects_imported_from_modeling: set[str] | None = None
) -> set:
    """Create augmented dependencies for a class node based on a `mapper`.
    Augmented dependencies means immediate dependencies + recursive function and assignments dependencies.
    """
    temp_module = cst.Module(body=[node])
    visitor = ClassDependencyMapper(node.name.value, set(mapper.global_nodes.keys()), objects_imported_from_modeling)
    temp_module.visit(visitor)


SUMMARY:
**Summary:**

`augmented_dependencies_for_class_node` function:
 Creates augmented dependencies for a class node by traversing its members and recursively visiting dependent modules.

**Key Highlights:**

* Takes in `cst.ClassDef`, `ModuleMapper`, and `objects_imported_from_modeling` parameters.
* Visits the class node using a custom `ClassDependencyMapper` visitor, which collects dependencies.
* Returns augmented dependencies via the ` augment_dependencies` method of the `ModuleMapper`.

**Implementation Details:**

* Creates a temporary module with the class node as its body.
* Uses the `ClassDependencyMapper` to recursively traverse dependent modules.
* Augments the collected dependencies using the `augment_dependencies` method.


key: chroma:document
string_value:
CODE:

class ModuleMapper(CSTVisitor, ABC):
    """An abstract visitor class which analyses a module, creating a mapping of dependencies for classes, functions and assignments.
    Class dependencies are computed with `compute_class_dependencies()`, while function and assignment dependencies are stored in
    `self.object_recursive_dependency_mapping` (can be computed by `_compute_recursive_object_dependencies()`).
    It defines common visiting patterns (i.e. common visit_xxx/leave_xxx functions) between the modular file and the
    modeling files that will be visited.
    """

    METADATA_DEPENDENCIES = (ParentNodeProvider, PositionProvider)

    def __init__(self, python_module: cst.Module):
        # fmt: off
        self.python_module: cst.Module = python_module             # original cst.Module being visited
        self.classes: Dict[str, cst.ClassDef] = {}                 # mapping from class names to Nodes (it will be ordered by default!!)
        self.imports = []                                          # stores all import statements
        self.functions: Dict[str, cst.FunctionDef] = {}            # mapping of global scope function names to Nodes
        self.object_dependency_mapping = defaultdict(set)          # immediate function/assignment dependency mapping (i.e. dependencies immediately in the function/assignment definition)
        self.assignments: Dict[str, cst.SimpleStatementLine] = {}  # mapping of global assignments names to Nodes
        self.current_function = None                               # this keeps track of the current module-scope function
        self.current_assignment = None                             # this keeps track of the current module-scope assignment
        # this keeps track of objects imported from modeling files (`from .configuration import Config`) -> `Config` should not be a dependency
        self.objects_imported_from_modeling = set()
        # regex pattern joining every possible file type
        self.match_patterns = "|".join(ALL_FILE_TYPES)
        # fmt: on

    def visit_ImportFrom(self, node):
        """This keeps track of objects imported from neighbor modeling files (e.g. in `modeling_xxx.py, we have
        `from .configuration_xxx import Config`, then `Config` should be recorded as it is not a dependency that needs
        to be added (because it will be part of the imports)"""
        import_module = self.python_module.code_for_node(node.module)
        import_statement = "." * len(node.relative) + import_module
        if re.search(rf"^\.({self.match_patterns})_.*", import_statement):
            for imported_object in node.names:
                # If an alias is present, we record it and not the original name
                if imported_object.evaluated_alias is not None:
                    self.objects_imported_from_modeling.add(imported_object.evaluated_alias)
                else:
                    self.objects_imported_from_modeling.add(imported_object.evaluated_name)

    def visit_SimpleStatementLine(self, node):
        """
        Global Assigns like `GEMMA_INPUT_DOCSTRING = 'THIS IS THE INPUT'` and all import statements
        are extracted and saved in their corresponding dict. They are then used when updating dependency mappings.
        """
        parent_node = self.get_metadata(cst.metadata.ParentNodeProvider, node)
        simple_top_level_assign_structure = m.SimpleStatementLine(
            body=[m.Assign(targets=[m.AssignTarget(target=m.Name())])]
        )
        if m.matches(parent_node, m.Module()):
            if m.matches(node, simple_top_level_assign_structure):
                left_hand_side = node.body[0].targets[0].target.value
                self.current_assignment = left_hand_side
                self.assignments[left_hand_side] = node
            elif m.matches(node, m.SimpleStatementLine(body=[m.Import() | m.ImportFrom()])):
                self.imports.append(node)

    def leave_SimpleStatementLine(self, node):
        # No need to check for the parent here -> everytime we exit one, it should be None anyway independently of where the
        # SimpleStatement is located
        self.current_assignment = None

    def visit_FunctionDef(self, node):
        parent_node = self.get_metadata(cst.metadata.ParentNodeProvider, node)
        if m.matches(parent_node, m.Module()):
            self.current_function = node.name.value
            self.functions[node.name.value] = node

    def leave_FunctionDef(self, node):
        parent_node = self.get_metadata(cst.metadata.ParentNodeProvider, node)
        if m.matches(parent_node, m.Module()):
            self.current_function = None

    def visit_If(self, node):
        for stmt in node.body.body:
            if m.matches(stmt, m.SimpleStatementLine(body=[m.ImportFrom() | m.Import()])):
                self.imports.append(node)

    def visit_ClassDef(self, node: ClassDef) -> None:
        """Record class nodes to create their dependencies at the end."""
        self.classes[node.name.value] = node

    def visit_Name(self, node: cst.Call):
        """This is used to create a mapping from module-scope functions and assignments to objects used inside them."""
        if self.current_function is not None:
            self.object_dependency_mapping[self.current_function].add(node.value)
        if self.current_assignment is not None:
            self.object_dependency_mapping[self.current_assignment].add(node.value)

    def leave_Module(self, node):
        """When leaving the module, we store the position of each global scoped node to allow sorting the dependencies
        based on their position in the code later. We use the PositionProvider metadata wrapper for this.
        We also make sure to update `self.object_dependency_mapping` so that it contains only names recorded in
        `self.global_nodes`.
        """
        # assign all nodes
        self.global_nodes = {**self.assignments, **self.classes, **self.functions}
        # now sort the class dependency_mapping based on the position of the nodes
        self.start_lines = {}
        for id, node in self.global_nodes.items():
            self.start_lines[id] = self.get_metadata(cst.metadata.PositionProvider, node).start.line

    def _restrict_dependencies_to_known_entities(self):
        """Since we added every Name as part of `self.object_dependency_mapping`, we need to remove those that
        are not part of the recorded objects in `self.global_nodes` (i.e. built-in variables, imports, etc).
        This should be called only after all merging operations have been finalized!!"""
        global_objects = set(self.global_nodes.keys())
        for object_name, dependencies in self.object_dependency_mapping.items():
            self.object_dependency_mapping[object_name] = {dep for dep in dependencies if dep in global_objects}

    def _compute_recursive_object_dependencies(self) -> dict[str, set]:
        """Based on immediate dependency mapping, create the recursive dependency mapping. For example, given the
        following file:
        ```
        def foo():
            pass

        def bar():
            foo()

        def test():
            bar()
        ```
        this visitor can only record immediate dependencies, i.e. it will record the following
        `self.object_dependency_mapping = {"test": {"bar"}, "bar": {"foo}}`. This function is used to create
        the recursive mapping, i.e. `recursive_dependencies = {"test": {"bar", "foo"}, "bar": {"foo}}`.
        """
        recursive_dependencies = {}
        for object_name in self.object_dependency_mapping.keys():
            all_dependencies = find_all_dependencies(self.object_dependency_mapping, start_entity=object_name)
            recursive_dependencies[object_name] = all_dependencies
        return recursive_dependencies

    def augment_dependencies(self, dependencies: set[str]) -> set[str]:
        """For a set of `dependencies`, augment them by adding all potential dependencies of the **functions** and
        **assignments** present in the `dependencies`.
        """
        new_dependencies = dependencies.copy()
        # Go through the set of dependencies
        for dep in tuple(dependencies):
            if dep in self.object_recursive_dependency_mapping.keys():
                new_dependencies.update(self.object_recursive_dependency_mapping[dep])
        return new_dependencies

    def compute_class_dependencies(self):
        """For each visited class, find its dependencies based on visiting the current file + potential merged dependencies."""
        self.class_dependency_mapping = {}
        for class_name, class_node in self.classes.items():
            dependencies = dependencies_for_class_node(class_node, set(self.global_nodes.keys()))
            # Correctly augment class dependencies with all needed objects
            self.class_dependency_mapping[class_name] = self.augment_dependencies(dependencies)

    @abstractmethod
    def compute_relative_order(self, missing_dependencies: set) -> dict[str, int]:


SUMMARY:
**ModuleMapper Summary**

The `ModuleMapper` class is an abstract visitor that analyzes a Python module, creating a mapping of dependencies for classes, functions, and assignments. It computes dependencies using `compute_class_dependencies()` and `_compute_recursive_object_dependencies()`, storing them in `self.object_recursive_dependency_mapping`.

**Main Purpose and Functionality:**

* Analyzes a Python module to create a dependency mapping
* Computes class dependencies using `compute_class_dependencies()`
* Stores object dependencies using `_compute_recursive_object_dependencies()`
* Augments dependencies using `augment_dependencies()` and `dependencies_for_class_node()`

**Class Relationship Mapping:**

* Records class nodes in `self.classes` for future dependencies
* Maps functions and assignments to objects used inside them using `visit_Name()` and `visit_ClassDef()`
* Updates `self.object_recursive_dependency_mapping` with recursive dependencies

**Dependencies Computation:**

* Computes absolute dependencies using `_compute_recursive_object_dependencies()`
* Augments dependencies with potential dependencies of functions and assignments

**Abstract Methods:**

* `compute_relative_order(missing_dependencies: set)`: returns a dictionary mapping class names to their relative order for missing dependencies.

Note: The code uses various internal classes, methods, and metadata wrappers that are not included in this summary.


key: chroma:document
string_value:
CODE:

class ModelFileMapper(ModuleMapper):
    """A mapper designed to parse modeling files (like `modeling_llama.py`). When encountering such a file
    in the `modular_xxx.py` file, we need to correctly visit it and merge the dependencies of the modular and current file.
    For this reason, this class should only be instantiated from the class method `visit_and_merge_dependencies`, which takes
    care of correctly merging dependencies, then finalizes all dependency graph computations.
    Note that we only merge functions and assignments here, as classes will be treated later on as they may be modified.
    For example, if you redefine `apply_rotary_pos_emb()` in the modular, the new node should be used in the dependencies
    of the modeling files as well.
    """

    def __init__(self, python_module: cst.Module):
        super().__init__(python_module)

    def compute_relative_order(self, missing_dependencies: set[str]) -> dict[str, int]:
        """Compute in which relative order the `missing_dependencies` should appear when the nodes are added to the final file that
        will be created based on the modular.
        """
        relative_order = {}
        idx = 0
        classes = sorted(
            [dep for dep in tuple(missing_dependencies) if dep in self.classes], key=lambda x: self.start_lines[x]
        )
        # This is because for merged dependencies, we only have relative order in the other visited file, so we need
        # to track dependency order relative to a given class
        if len(classes) > 0 and not hasattr(self, "class_dependency_mapping"):
            raise ValueError("Cannot correctly find the relative order of the dependencies.")

        remaining_dependencies = missing_dependencies.copy()

        # Start by tracking relative order class by class
        for class_name in classes:
            class_dependencies = tuple(self.class_dependency_mapping[class_name] & remaining_dependencies)
            original_dependencies = []
            merged_dependencies = []
            # We need to differentiate between nodes that were already present (we can get relative order globally) and
            # nodes that were merged (we can get relative order only relative to the class the dependencies relate to)
            for class_dep in class_dependencies:
                if class_dep in self.start_lines:
                    original_dependencies.append(class_dep)
                else:
                    merged_dependencies.append(class_dep)
            # Sort both list according to the order in their respective file
            original_dependencies = sorted(original_dependencies, key=lambda x: self.start_lines[x])
            merged_dependencies = sorted(merged_dependencies, key=lambda x: self.modular_file_start_lines[x])

            # Add all original node first, then merged ones
            for dep in original_dependencies + merged_dependencies:
                remaining_dependencies.remove(dep)
                relative_order[dep] = idx
                idx += 1
            # Add the class itself (it can sometimes already be present if the order of classes in the source file
            # does not make sense, i.e. a class is used somewhere before being defined like in `rt_detr`...)
            if class_name in remaining_dependencies:
                remaining_dependencies.remove(class_name)
                relative_order[class_name] = idx
                idx += 1

        # Now add what still remains
        remaining_dependencies = tuple(remaining_dependencies)
        original_dependencies = []
        merged_dependencies = []
        for dep in remaining_dependencies:
            if dep in self.modular_file_start_lines:
                merged_dependencies.append(dep)
            else:
                original_dependencies.append(dep)
        # Sort both list according to the order in their respective file
        original_dependencies = sorted(original_dependencies, key=lambda x: self.start_lines[x])
        merged_dependencies = sorted(merged_dependencies, key=lambda x: self.modular_file_start_lines[x])

        # Add all original node first, then merged ones
        for dep in original_dependencies + merged_dependencies:
            relative_order[dep] = idx
            idx += 1

        return relative_order

    def _merge_functions(self, functions: dict[str, cst.CSTNode], object_mapping: dict[str, set]):
        """Update the global nodes and function dependency mapping with those from the modular file.

        Merging rule: if any function with the same name was redefined in the modular, use it and its dependencies
        instead of the original ones (this may mean to add new functions as well, if any redefined function uses a new one).
        """
        # Add/overwrite all needed function nodes and dependencies
        self.functions.update(functions)
        self.object_dependency_mapping.update(
            {obj: dep for obj, dep in object_mapping.items() if obj in functions.keys()}
        )
        # Add them to global nodes
        self.global_nodes.update(self.functions)

    def _merge_assignments(self, assignments: dict[str, cst.CSTNode], object_mapping: dict[str, set]):
        """Update the global nodes with the assignment from the modular file.

        Merging rule: if any assignment with the same name was redefined in the modular, we use it and its dependencies ONLY if it is
        in `ASSIGNMENTS_TO_KEEP`. Otherwise, we use the original value and dependencies. This rule was chosen to avoid having to rewrite the
        big docstrings.
        """
        for assignment, node in assignments.items():
            if assignment in ASSIGNMENTS_TO_KEEP or assignment not in self.assignments:
                self.assignments[assignment] = node
                if assignment in object_mapping:
                    self.object_dependency_mapping[assignment] = object_mapping[assignment]
        # Add them to global nodes
        self.global_nodes.update(self.assignments)

    def _merge_classes(self, classes: dict[str, cst.CSTNode]):
        """Update the global nodes with the new classes from the modular (i.e. classes which do not exist in current file, and
        are not imported). We do NOT update any dependency mapping here. This is because we only need the names of newly defined
        classes in the modular to be discoverable when computing dependencies for new nodes later on. For this reason, we
        do not add the new classes to `self.classes`, but only to `global_nodes`.
        """
        # Add/overwrite all needed function nodes and dependencies
        self.global_nodes.update(
            {
                name: node
                for name, node in classes.items()
                if name not in self.classes and name not in self.objects_imported_from_modeling
            }
        )

    def merge_modular_dependencies(self, classes, functions, assignments, object_mapping, start_lines):
        """Merge classes, functions and assignments from the modular definitions into the current module file,
        then record the relative order of all nodes.
        Note: This function takes care of updating `global_nodes` and `object_recursive_dependency_mapping` as well after the
        merge with other files dependencies.
        """
        self._merge_functions(functions, object_mapping)
        self._merge_assignments(assignments, object_mapping)
        self._merge_classes(classes)
        self.modular_file_start_lines = start_lines

        # Restrict the dependency mappings to the known entities to avoid Python's built-ins and imports
        self._restrict_dependencies_to_known_entities()
        # Create the global mapping of recursive dependencies for functions and assignments
        self.object_recursive_dependency_mapping = self._compute_recursive_object_dependencies()

    @classmethod
    def visit_and_merge_dependencies(
        cls, module: cst.Module, classes, functions, assignments, object_mapping, start_lines
    ) -> "ModelFileMapper":
        wrapper = MetadataWrapper(module)
        mapper = cls(module)
        wrapper.visit(mapper)
        # Merge dependencies
        mapper.merge_modular_dependencies(classes, functions, assignments, object_mapping, start_lines)
        # Create the class dependencies graph
        mapper.compute_class_dependencies()


SUMMARY:
**ModelFileMapper Class Summary**

The `ModelFileMapper` class is designed to parse modeling files and merge their dependencies with those of the modular file.

### Purpose
To correctly visit and merge dependencies between modules, ensuring that functions, assignments, and classes are properly updated and ordered relative to each other.

### Key Implementation Details

*   **`merge_modular_dependencies` method**: Merges classes, functions, and assignments from the modular definitions into the current module file.
*   **`compute_relative_order` method**: Computes the relative order of all nodes based on their dependencies and ordering conventions in the source code.
*   **Methods for merging specific node types (functions, assignments, classes)**
*   **Updating global nodes and dependency mappings** after merging with modular file dependencies

### Notable Dependencies or Requirements

*   `cst` module for parsing Python code
*   `MetadataWrapper` class for accessing metadata of the parsed module
*   External dependencies are restricted to known entities using `_restrict_dependencies_to_known_entities` method.

### Class Inheritance and Interfaces

None, as it is a custom implementation.

**Example Usage**

```python
import cst

# Sample module definition
module_def = """class MyClass:
    def __init__(self):
        pass"""

# Create an instance of the ModelFileMapper class
mapper = ModelFileMapper(module)  # Assuming 'module' is a cst module object

# Call the visit_and_merge_dependencies method to process the module
mapper.visit_and_merge_dependencies(...
```

This code snippet provides a basic overview of how to use the `ModelFileMapper` class.


key: chroma:document
string_value:
CODE:

def common_partial_suffix(str1: str, str2: str) -> str:
    """Return the biggest common suffix between 2 strings. If one string is a full suffix of the other string,
    we do not consider it a common suffix and return `""`"""
    common_suffix = ""
    for i in range(1, min(len(str1), len(str2)) + 1):
        if str1[-i] == str2[-i]:
            common_suffix = str1[-i] + common_suffix
        else:
            break
    # We do not allow full string suffix
    if common_suffix == str1 or common_suffix == str2:
        common_suffix = ""


SUMMARY:
**Summary:**

### Function Name:
`common_partial_suffix`

### Primary Purpose:
Returns the largest common partial suffix between two input strings.

### Key Implementation Details:

* Iterates from the shortest string's end to find matching characters.
* Builds the `common_suffix` by prepending matching characters from `str1`.
* If one string is a full suffix of the other, returns an empty string.

### Notable Dependencies/Requirements:

* No external dependencies mentioned.
* The function uses built-in Python string concatenation and iteration.


key: chroma:document
string_value:
CODE:

def replace_class_node(
    mapper: ModelFileMapper, class_node: cst.ClassDef, renamed_super_class: str, original_super_class: str
):
    """
    Replace a class node which inherits from another modeling class. This function works in the following way:
    - start from the base class node of the inherited class (a cst.Node)
    - replace all methods of the base node with the methods defined in the child class
    - append all new methods defined in the child class
    - replace all calls to super() with the unravelled code

                    |    ```python                          |               |    ```python
                    |    class GemmaModel(LlamaModel):      |               |       class GemmaModel(nn.Module):
                    |        def __init__(self):            |               |           def __init__(self):
    Going from:     |            super().__init__()         |       to:     |               super().__init__(config)
                    |            self.dropout = 0.2         |               |               self.dropout = 0.2
                    |     ```                               |               |               self.padding_idx = config.pad_token_id
                                                                            |               self.vocab_size = config.vocab_size
                                                                            |               self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
                                                                            |               self.layers = nn.ModuleList(
                                                                            |                   [LlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
                                                                            |               )
                                                                            |               self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
                                                                            |               self.gradient_checkpointing = False
                                                                            |               # Initialize weights and apply final processing
                                                                            |               self.post_init()
                                                                            |     ```
    """
    all_bases = [get_full_attribute_name(k.value) for k in class_node.bases]
    if any(base is None for base in all_bases):
        raise ValueError(f"Could not parse the name of the bases for {class_node.name.value}")

    original_node = mapper.classes[renamed_super_class]
    # Always use the new name of the class (in case we use e.g. `ColPaliForRetrieval` inheriting from `PaliGemmaForConditionalGeneration`)
    new_name = class_node.name

    # If the new class name is different from the renamed super class name, we need to update the docstrings/comments accordingly
    if new_name.value != renamed_super_class:
        common_suffix = common_partial_suffix(new_name.value, renamed_super_class)
        # Note that this works even without common prefix, in which case it does not replace anything
        old, new = renamed_super_class.replace(common_suffix, ""), new_name.value.replace(common_suffix, "")
        temp_module = cst.Module(body=[original_node])
        original_node = temp_module.visit(
            ReplaceNameTransformer(get_lowercase_name(old), get_lowercase_name(new), only_doc=True)
        ).body[0]

    # If we explicitly passed a new base with common suffix to an old base, it is for switching the prefix
    # e.g. if the "natural" parent class is `PreTrainedModel` but we wanted to rename it to `PreTrainedVisionModel`
    additional_bases = [base for base in all_bases if base != original_super_class]
    new_bases = []
    for original_base in original_node.bases:
        new_base = original_base
        # we only potentially switch base for Name-based bases, not Attribute
        if m.matches(original_base.value, m.Name()):
            original_base_name = original_base.value.value
            for additional_base_name in additional_bases:
                suffix = common_partial_suffix(original_base_name, additional_base_name)
                if len(suffix) > 0 and suffix[0].isupper():
                    new_name_node = original_base.value.with_changes(value=additional_base_name)
                    new_base = original_base.with_changes(value=new_name_node)
                    break
        new_bases.append(new_base)

    original_methods = {
        f.name.value if hasattr(f, "name") else mapper.python_module.code_for_node(f): f
        for f in original_node.body.body
    }
    updated_methods = {
        f.name.value if hasattr(f, "name") else mapper.python_module.code_for_node(f): f for f in class_node.body.body
    }
    end_meth = []

    assign_targets = {}
    docstring_node = []
    # Iterate directly from node.body as there can be property/setters with same names which are overwritten when we use a dict
    for func in original_node.body.body:
        name = func.name.value if hasattr(func, "name") else mapper.python_module.code_for_node(func)
        if m.matches(func, m.FunctionDef()) and name in updated_methods and updated_methods[name] is not None:
            new_params = updated_methods[name].params
            # Replace the method in the replacement class, preserving decorators
            kwarg_name = getattr(updated_methods[name].params, "star_kwarg", None)
            if kwarg_name and kwarg_name.name.value == "super_kwargs":
                parent_params = {k.name.value: k for k in func.params.params}
                parent_params.update({k.name.value: k for k in new_params.params[1:]})
                new_params = new_params.with_changes(
                    params=list(parent_params.values()), star_kwarg=func.params.star_kwarg
                )
            # Keep decorators in `modular_xxx.py` if any, else original decorators
            new_decorators = (
                updated_methods[name].decorators if len(updated_methods[name].decorators) > 0 else func.decorators
            )
            if not re.match(
                r"\ndef .*\(.*\):\n    raise.*Error\(.*",
                mapper.python_module.code_for_node(updated_methods[name]),
            ):
                func = func.with_changes(body=updated_methods[name].body, params=new_params, decorators=new_decorators)
            else:
                continue

        if m.matches(func, m.SimpleStatementLine(body=[m.Assign()])):
            target = mapper.python_module.code_for_node(func.body[0].targets[0])
            assign_targets[target] = func
        elif m.matches(func, m.SimpleStatementLine(body=[m.AnnAssign()])):
            target = mapper.python_module.code_for_node(func.body[0].target)
            assign_targets[target] = func
        elif m.matches(func, DOCSTRING_NODE):
            docstring_node = [func]
        else:
            end_meth.append(func)

    # Port new methods that are defined only in modular-file and append at the end
    for func in class_node.body.body:
        name = func.name.value if hasattr(func, "name") else mapper.python_module.code_for_node(func)
        if m.matches(func, DOCSTRING_NODE):  # This processes the docstring of the class!
            # Extract the original docstring
            updated_docstring = func.body[0].value.value
            if len(docstring_node) == 0:  # If the original docstring is empty, just create one from the updated.
                docstring_node = [
                    cst.SimpleStatementLine(body=[cst.Expr(value=cst.SimpleString(value=updated_docstring))])
                ]
            else:
                original_docstring = docstring_node[0].body[0].value.value
                merged_doc = merge_docstrings(original_docstring, updated_docstring)
                # Update the docstring in the original function
                docstring_node = [
                    docstring_node[0].with_changes(body=[cst.Expr(value=cst.SimpleString(value=merged_doc))])
                ]
        if name not in original_methods and func is not None and isinstance(func, cst.FunctionDef):
            end_meth.append(func)
        if m.matches(func, m.SimpleStatementLine(body=[m.Assign()])):
            # TODO we only use single assign might cause issues
            target = mapper.python_module.code_for_node(func.body[0].targets[0])
            assign_targets[target] = func
        if m.matches(func, m.SimpleStatementLine(body=[m.AnnAssign()])):
            target = mapper.python_module.code_for_node(func.body[0].target)
            assign_targets[target] = func
    end_meth = docstring_node + list(assign_targets.values()) + end_meth

    # Replace the calls to `super()` with the unrolled code
    result_node = original_node.with_changes(body=cst.IndentedBlock(body=end_meth))
    temp_module = cst.Module(body=[result_node])
    new_module = MetadataWrapper(temp_module)
    new_replacement_class = new_module.visit(
        SuperTransformer(temp_module, original_methods, updated_methods, all_bases)
    )
    new_replacement_body = new_replacement_class.body[0].body  # get the indented block

    # Use decorators redefined in `modular_xxx.py` if any
    new_decorators = class_node.decorators if len(class_node.decorators) > 0 else original_node.decorators

    return original_node.with_changes(
        body=new_replacement_body, decorators=new_decorators, bases=new_bases, name=new_name


SUMMARY:
This function replaces a Python class node in an abstract syntax tree (AST) with a new implementation. The replacement process involves:

1. Updating the base classes and names of the class.
2. Replacing methods defined in the base classes with those defined in the child class.
3. Appending new methods not present in the parent class but present in the child class.
4. Replacing calls to `super()` with the updated code.

The function takes as input a model file mapper, a node representing the class to be replaced, and the names of the superclasses and the child's superclass. It returns a new node representing the modified class.


key: chroma:document
string_value:
CODE:

def find_file_type(class_name: str) -> str:
    """Based on a class name, find the file type corresponding to the class.
    If the class name is `LlamaConfig` it will return `configuration`.
    The list of suffixes is in `TYPE_TO_FILE_TYPE`. If there are no match, we match by default to `modeling`
    """
    match_pattern = "|".join(TYPE_TO_FILE_TYPE.keys())
    match = re.search(rf"({match_pattern})$", class_name)
    if match:
        file_type = TYPE_TO_FILE_TYPE[match.group(1)]
    else:
        file_type = "modeling"


SUMMARY:
**Summary:**

### Function Purpose and Functionality

`find_file_type` function takes a `class_name` string as input and returns the corresponding file type. It uses a lookup table `TYPE_TO_FILE_TYPE` to match class names against predefined file types.

### Key Implementation Details

* Uses regular expressions to search for matching file types
* Falls back to "modeling" if no match is found in `TYPE_TO_FILE_TYPE`

### Notable Dependencies and Requirements

* `re` (regular expression) module
* `TYPE_TO_FILE_TYPE` dictionary (not shown in this snippet)

### Method Signatures and Interfaces

* `find_file_type(class_name: str) -> str`: returns a file type string based on the input class name


key: chroma:document
string_value:
CODE:

def append_new_import_node(
    node: cst.CSTNode, unused_imports: set[str], added_names: set, imports_to_keep: list[cst.CSTNode]
):
    """Insert the new `node` to the list of `imports_to_keep` in-place, if it is not part of the `unused_imports` or `added_names`.
    Also modifies `added_names` in-place accordingly."""
    import_node = node.body[0]
    names_to_keep = []
    for name in import_node.names:
        name_value = name.evaluated_name
        if name_value not in unused_imports and name_value not in added_names:
            names_to_keep.append(name.with_changes(comma=cst.MaybeSentinel.DEFAULT))
            added_names.add(name_value)
    if len(names_to_keep) > 0:
        new_node = node.with_changes(body=[import_node.with_changes(names=names_to_keep)])


SUMMARY:
**Summary of `append_new_import_node` function**

Inserts a new import node (`node`) to the `imports_to_keep` list if it's not part of unused or added imports. Modifies `added_names` accordingly.

**Key Implementation Details:**

* Inserts `import_node` into `imports_to_keep` with filtered names.
	+ `names_to_keep = [name_value]` where `name_value != unused_imports` and `name_value != added_names`.
* Applies modifications to `node` using:
	+ `with_changes(body=[import_node.with_changes(names=names_to_keep)])`
	+ `imports_to_keep.append(new_node)`.

**Notable Dependencies/Requirements:**

* `cst.CSTNode`: Input node type.
* `added_names`: Set of names to keep track of.
* `unused_imports`: Set of unused imports.

**External Dependency:**

* `cst.MaybeSentinel.DEFAULT`.


key: chroma:document
string_value:
CODE:

def get_needed_imports(body: dict[str, dict], all_imports: list[cst.CSTNode]) -> list[cst.CSTNode]:
    """Get all the imports needed in the `body`, from the list of `all_imports`.
    `body` is a dict with the following structure `{str: {"insert_idx": int, "node": cst.CSTNode}}`.
    Note: we need to use `isinstance` on scope assignements, m.matches apparently does not work here yet!
    """
    new_body = [k[1]["node"] for k in sorted(body.items(), key=lambda x: x[1]["insert_idx"])]
    wrapper = MetadataWrapper(cst.Module(body=all_imports + new_body))
    scopes = set(wrapper.resolve(ScopeProvider).values())
    unused_imports = set()
    import_ref_count = defaultdict(lambda: 0)
    for scope in scopes:
        for assignment in scope.assignments:
            node = assignment.node
            if isinstance(assignment, cst.metadata.Assignment) and isinstance(node, (cst.Import, cst.ImportFrom)):
                ref_count = len(assignment.references)
                name = assignment.name
                import_ref_count[name] = max(ref_count, import_ref_count[name])
    # Similar imports may be redefined, and only used between their 1st and 2nd definition so if we already have
    # a ref count > 0 at any point, the imports is actually used
    unused_imports = {name for name, count in import_ref_count.items() if count <= 0 or name in body.keys()}

    imports_to_keep = []
    # We need to keep track of which names were already imported, because some import may be duplicated from multiple sources
    # or be both protected and unprotected due to inconsistency between models
    added_names = set()
    existing_protected_statements = set()  # str repr of the import nodes - does not work with the nodes directly
    for node in all_imports:
        if m.matches(node, m.If()):  # handle safe imports
            new_statements = []
            for stmt_node in node.body.body:
                append_new_import_node(stmt_node, unused_imports, added_names, new_statements)
            new_statements = [stmt for stmt in new_statements if str(stmt) not in existing_protected_statements]
            if len(new_statements) > 0:
                new_node = node.with_changes(body=node.body.with_changes(body=new_statements))
                imports_to_keep.append(new_node)
                existing_protected_statements.update({str(stmt) for stmt in new_statements})
        else:
            append_new_import_node(node, unused_imports, added_names, imports_to_keep)

    protected_import_nodes = [node for node in imports_to_keep if m.matches(node, m.If())]
    usual_import_nodes = [node for node in imports_to_keep if not m.matches(node, m.If())]

    # Protected imports always appear at the end of all imports


SUMMARY:
**Function Summary: `get_needed_imports`**

*   Purpose:
    *   Generates needed imports for a given code segment (`body`) from a list of provided imports (`all_imports`)
*   Functionality:

    1.  **Sorting and Preparation**: Sorts the `body` dictionary based on insert indices, creates a metadata wrapper around the combined `all_imports + new_body`, and resolves scopes to identify assignments.
    2.  **Import Ref Counting**: Iterates through scope assignments to count reference counts for each import name.
    3.  **Unused Imports Detection**: Identifies unused imports based on ref count threshold and existing references in the code segment (`body`).
*   Output:
    *   A list of needed imports, separated into `usual` (non-protected) and `protected` nodes.

**Key Implementation Details:**

*   Uses a dictionary-based approach to sort and prepare the input data.
*   Employs a metadata wrapper to analyze import statements.
*   Utilizes scope resolution to identify assignments and determine ref counts.
*   Applies thresholds for identifying unused imports.

**Notable Dependencies/Requirements:**

*   `cst.CSTNode` (used for parsing and analyzing code structure)
*   `MetadataWrapper` (custom class used for metadata-driven analysis)
*   `ScopeProvider` (interface for scope resolution)
*   Custom modules (`m.matches`, `append_new_import_node`) not explicitly referenced in the provided function definition.


key: chroma:document
string_value:
CODE:

def split_all_assignment(node: cst.CSTNode) -> dict[str, cst.CSTNode]:
    """Split the `__all__` assignment found in the modular between each corresponding files."""
    all_all_per_file = {}
    assign_node = node.body[0]
    if isinstance(assign_node.value, cst.List):
        # Extract the elements from the list
        all_all_to_add = defaultdict(list)
        for element in assign_node.value.elements:
            if isinstance(element.value, cst.SimpleString):
                # Remove quotes and add the string to the elements list
                class_name = element.value.value
                file = find_file_type(element.value.evaluated_value)
                all_all_to_add[file] += [class_name]
        for file, new_alls in all_all_to_add.items():
            new_node = assign_node.with_changes(
                value=cst.List(elements=[cst.Element(value=cst.SimpleString(value=k)) for k in new_alls])
            )
            all_all_per_file[file] = node.with_changes(body=[new_node])


SUMMARY:
**Function Summary: `split_all_assignment`**

**Purpose:** Splits the `__all__` assignment in a modular into separate assignments for corresponding files.

**Main Functionality:**
Extracts class names from the list of values assigned to `__all__`, determines the file type, and regroups them by file using a dictionary. This function splits the initial assignment into multiple assignments, one per file.

**Key Implementation Details:**

* Iterates through a list of values (`node.body[0].value.elements`) and extracts class names from strings.
* Determines the file type for each class name using `find_file_type(element.value.evaluated_value)`.
* Regroups class names by file using a dictionary (`all_all_to_add`).

**Method Signatures and Interfaces:**

* `assign_node.with_changes`: Replaces the value of an assignment node with a new list.
* `node.with_changes`: Wraps the entire node with new body.

**Notable Dependencies and Requirements:**

* `cst.CSTNode`
* ` defaultdict(list)`
* `find_file_type` (not provided)


key: chroma:document
string_value:
CODE:

class ModularFileMapper(ModuleMapper):
    """This is a Mapper to visit a modular file (like `modular_llama.py`). It visits the whole file, recording dependency,
    then visits all imported modeling files (like `modeling_llama.py`), and manages their mutual dependencies.
    Calling the method `create_modules()` after visit will create all modules based on this modular file.
    """

    def __init__(self, python_module, new_name):
        super().__init__(python_module)
        # fmt: off
        self.model_name = new_name  # name of the model being defined. Should be in the format of `llama` or `layout_xlm` or `phi3`

        self.model_specific_imported_objects: Dict[str, str] = {}  # e.g. {"LlamaModel": "transformers.models.llama.modeling_llama"}
        self.model_specific_modules: Dict[str, cst.Module] = {}  # e.g. {"transformers.models.llama.modeling_llama": cst.Module}

        self.all_all_to_add = {}
        # fmt: on

    def visit_ImportFrom(self, node: cst.ImportFrom) -> None:
        """When visiting imports from modeling files (i.e. `transformers.models.xxx`) we get the code, parse it,
        and save it in `self.model_specific_modules` to later visit. The imported objects are saved in `self.model_specific_imported_objects`.
        """
        import_module = self.python_module.code_for_node(node.module)
        import_statement = "." * len(node.relative) + import_module
        if any(import_to_skip in import_statement for import_to_skip in IMPORTS_TO_SKIP_IN_MODULAR):
            return
        if m.matches(node.module, m.Attribute()):
            for imported_ in node.names:
                _import = re.search(
                    rf"(?:transformers\.models\.)|(?:\.\.)\w+\.({self.match_patterns})_.*", import_statement
                )
                if _import:
                    source = _import.group(1)
                    if source == "modeling" and "Config" in self.python_module.code_for_node(imported_):
                        raise ValueError(
                            f"You are importing {self.python_module.code_for_node(imported_)} from the modeling file. Import from the `configuration_xxxx.py` file instead"
                        )
                    if import_module not in self.model_specific_modules:
                        if "models" not in import_module:
                            import_module = "models." + import_module
                        if "transformers" not in import_module:
                            import_module = "transformers." + import_module
                        source_code = get_module_source_from_name(import_module)
                        tree = cst.parse_module(source_code)
                        self.model_specific_modules[import_module] = tree
                    imported_object = self.python_module.code_for_node(imported_.name)
                    self.model_specific_imported_objects[imported_object] = import_module
        if m.matches(node.module, m.Name()):
            if "transformers" == import_module:
                raise ValueError(
                    f"You are importing from {import_module} directly using global imports. Import from the correct local path"
                )

    def visit_SimpleStatementLine(self, node):
        """If we visit an import statement not previously visited, record it. If we visit a module-scope assignment,
        simply record it or, if it is `__all__`, split it between files where we should dispatch it.
        """
        parent_node = self.get_metadata(cst.metadata.ParentNodeProvider, node)
        simple_top_level_assign_structure = m.SimpleStatementLine(
            body=[m.Assign(targets=[m.AssignTarget(target=m.Name())])]
        )
        if m.matches(parent_node, m.Module()):
            if m.matches(node, m.SimpleStatementLine(body=[m.Import()])):
                self.imports.append(node)
            elif m.matches(node, m.SimpleStatementLine(body=[m.ImportFrom()])):
                import_module = self.python_module.code_for_node(node.body[0].module)
                import_statement = "." * len(node.body[0].relative) + import_module
                if not (
                    re.search(rf"(?:transformers\.models\.)|(?:\.\.)\w+\.({self.match_patterns})_.*", import_statement)
                    and not any(import_to_skip in import_statement for import_to_skip in IMPORTS_TO_SKIP_IN_MODULAR)
                ):
                    self.imports.append(node)
            elif m.matches(node, simple_top_level_assign_structure):
                assigned_variable = node.body[0].targets[0].target.value
                # __all__ is treated differently and not added to general assignments
                if assigned_variable == "__all__":
                    self.all_all_to_add = split_all_assignment(node)
                else:
                    self.current_assignment = assigned_variable
                    self.assignments[assigned_variable] = node

    def leave_Module(self, node):
        """When we leave the modular file, we do the following in order:
        1. for each modeling file found in the imports, rename it with the new model name, visit it, and update
        its dependency graph with the new function and assignment definitions found in the modular
        2. update the modular dependency graph with the imported functions and assignments (found when visiting the matching files)
        3. compute the nested (recursive) function and assignment dependencies
        """
        # Takes care of finalizing our visit
        super().leave_Module(node)

        # 1. for each modeling file found in the imports, rename it with the new model name, visit it, and update dependencies
        self.visited_modules = {}
        self.renamers = {}
        name_prefixes = self.infer_new_model_name()
        for file, module in self.model_specific_modules.items():
            file_model_name = file.split(".")[-2]
            new_name = name_prefixes[file]
            renamer = ReplaceNameTransformer(file_model_name, new_name, self.model_name)
            renamed_module = module.visit(renamer)
            self.visited_modules[file] = ModelFileMapper.visit_and_merge_dependencies(
                renamed_module,
                self.classes,
                self.functions,
                self.assignments,
                self.object_dependency_mapping,
                self.start_lines,
            )
            # We record it so that we can rename classes later the exact same way
            self.renamers[file] = renamer

        # 2. in turn, we need to add the imported functions/assignments to the dependencies of the modular mapper, using the
        # definitions found in the visited files
        self.merge_model_specific_imports(self.visited_modules)

        # 3. compute the nested (recursive) function and assignment dependencies
        self.object_recursive_dependency_mapping = self._compute_recursive_object_dependencies()

        # We need to keep track of which objects were imported directly into which modeling file to not add them wrongly later
        # Note that we may visit several of the same file types, thus we save them per file type, not file
        self.imported_objects_per_file = defaultdict(set)
        for file, mapper in self.visited_modules.items():
            file_type = re.search(rf"^transformers\.models\.\w+\.({self.match_patterns})_.*", file).group(1)
            self.imported_objects_per_file[file_type].update(mapper.objects_imported_from_modeling)

    def merge_model_specific_imports(self, visited_modules):
        """Merge the functions and assignments imported from the modeling files to the modular nodes and dependency graph,
        based on the visited files."""
        self.start_lines_file_mapping = {}
        self.added_objects_file_mapping = {}
        for object_name, file in self.model_specific_imported_objects.items():
            visited_module = visited_modules[file]
            self.start_lines_file_mapping[file] = visited_module.start_lines
            # Add functions and their dependencies
            if object_name in visited_module.functions and object_name not in self.functions:
                self.functions[object_name] = visited_module.functions[object_name]
                self.added_objects_file_mapping[object_name] = file
                dependencies = visited_module.object_dependency_mapping.get(object_name, None)
                if dependencies is not None:
                    self.object_dependency_mapping[object_name] = dependencies
                    for dep in dependencies:
                        if dep not in self.global_nodes:
                            self.added_objects_file_mapping[dep] = file
                            self.functions[dep] = visited_module.global_nodes[dep]

            # Add assignments and their dependencies
            elif object_name in visited_module.assignments and object_name not in self.assignments:
                self.assignments[object_name] = visited_module.assignments[object_name]
                self.added_objects_file_mapping[object_name] = file
                dependencies = visited_module.object_dependency_mapping.get(object_name, None)
                if dependencies is not None:
                    self.object_dependency_mapping[object_name] = dependencies
                    for dep in dependencies:
                        if dep not in self.global_nodes:
                            self.added_objects_file_mapping[dep] = file
                            self.assignments[dep] = visited_module.global_nodes[dep]

        # Do not forget to re-assign all nodes after the merge
        self.global_nodes = {**self.assignments, **self.classes, **self.functions}
        # And restric dependencies to those nodes only
        self._restrict_dependencies_to_known_entities()

    def compute_relative_order(self, missing_dependencies: set) -> dict[str, int]:
        """Compute in which relative order the `missing_dependencies` should appear when the nodes are added to the final file that
        will be created based on the modular.
        """
        relative_order = {}
        idx = 0

        original_dependencies = []
        other_files_dependencies = defaultdict(list)
        for dep in tuple(missing_dependencies):
            if dep in self.added_objects_file_mapping:
                file = self.added_objects_file_mapping[dep]
                other_files_dependencies[file].append(dep)
            else:
                original_dependencies.append(dep)
        # Sort all lists according to the order in their respective file
        all_dependencies = []
        for file, dependencies in other_files_dependencies.items():
            sorted_dependencies = sorted(dependencies, key=lambda x: self.start_lines_file_mapping[file][x])
            all_dependencies += sorted_dependencies
        all_dependencies += sorted(original_dependencies, key=lambda x: self.start_lines[x])

        # Add all original node first, then merged ones (one file at a time)
        for dep in all_dependencies:
            relative_order[dep] = idx
            idx += 1

        return relative_order

    def infer_new_model_name(self) -> dict:
        """Infer whether we are using a model name prefix different from the usual model name as defined from the filename.
        This is useful e.g. when we define a new multi-modal model, and only the text part inherits from `LlamaModel`,
        so we have something like:
        ```python
        class NewModelNameTextDecoderLayer(LlamaDecoderLayer):
            pass
        ```
        with the `Text` prefix added to the model name.
        However, in case of multiple prefix used, we raise a warning and use the most frequent prefix, to avoid parsing
        the same file multiple times and inconsistencies in the objects added from dependencies.
        If the new prefix collides with a prefix of another class in the file where we are importing from, then we also
        raise a warning, and use the default prefix (model name) to avoid collisions in dependencies.
        """
        prefix_model_name_mapping = defaultdict(Counter)
        cased_default_name = get_cased_name(self.model_name)
        # Iterate over all new classes to get modeling super classes
        for class_name, class_node in self.classes.items():
            modeling_bases = [
                k.value.value for k in class_node.bases if k.value.value in self.model_specific_imported_objects
            ]
            if len(modeling_bases) > 1:
                raise ValueError(
                    f"{class_name} was defined with more than 1 model-specific super class. This is unsupported. We found {*modeling_bases,}."
                )
            if len(modeling_bases) == 1:
                filename = self.model_specific_imported_objects[modeling_bases[0]]
                cased_model_name = cased_default_name  # the default name prefix
                suffix = common_partial_suffix(class_name, modeling_bases[0])
                if len(suffix) > 0 and suffix[0].isupper():
                    cased_model_name = class_name.replace(suffix, "")
                prefix_model_name_mapping[filename].update([cased_model_name])

        # Check if we found multiple prefixes for some modeling files
        final_name_mapping = {}
        for file, prefixes_counter in prefix_model_name_mapping.items():
            if len(prefixes_counter) > 1:
                _, total = prefixes_counter.most_common(1)[0]
                most_used_entities = [name for name, count in prefixes_counter.most_common() if count == total]
                # if the default name is in the pool of equally used prefixes, use it, otherwise last encountered
                final_name = cased_default_name if cased_default_name in most_used_entities else most_used_entities[-1]
            else:
                final_name = list(prefixes_counter)[0]
            # Check if the prefix can be used without collisions in the names
            old_cased_model_name = get_cased_name(file.split(".")[-2])
            old_model_name_prefix = final_name.replace(cased_default_name, old_cased_model_name)
            # Raise adequate warning depending on the situation
            has_prefix_collision = f"\nclass {old_model_name_prefix}" in get_module_source_from_name(file)
            if final_name != cased_default_name and has_prefix_collision:
                if len(prefixes_counter) > 1:
                    logger.warning(
                        f"We detected multiple prefix names when inheriting from {file}: {*set(prefixes_counter),}. However, the "
                        f"most used one, '{final_name}', is already present in the source file and will likely cause consistency "
                        f"issues. For this reason we fallback to the default prefix '{cased_default_name}' when grabbing args "
                        "and dependencies. Make sure to subclass the intermediate classes with the prefix you want (if different "
                        f"from '{cased_default_name}') or use a single prefix in all the modular (best)."
                    )
                else:
                    logger.warning(
                        f"We detected the use of the new default prefix {final_name} when inheriting from {file}. However, it is "
                        "already present in the source file and will likely cause consistency issues. For this reason we fallback "
                        f"to the default prefix '{cased_default_name}' when grabbing args and dependencies. Make sure to subclass "
                        f"the intermediate classes with the prefix you want (if different from '{cased_default_name}')"
                    )
                final_name = cased_default_name
            elif len(prefixes_counter) > 1:
                logger.warning(
                    f"We detected multiple prefix names when inheriting from {file}: {*set(prefixes_counter),}. We will only "
                    f"use the most used '{final_name}' prefix when grabbing args and dependencies. Make sure to subclass the "
                    f"intermediate classes with the prefix you want (if different from '{final_name}') or use a single prefix "
                    "in all the modular (best)."
                )
            final_name_mapping[file] = get_lowercase_name(final_name)

        # Check we are not missing imported files
        for file in self.model_specific_modules.keys():
            if file not in final_name_mapping.keys():
                final_name_mapping[file] = self.model_name



SUMMARY:
This is a Python code that appears to be a part of the `transformers` library, specifically handling model merging and renaming. Here's a concise summary:

The code defines several methods for merging models, renaming classes, and handling naming conflicts. It checks if multiple prefixes are used in a modeling file and attempts to use the most frequent or default prefix instead. If a collision is detected, it raises a warning and may use a fallback value.

Key features include:

* Merging models by checking common prefixes
* Renaming classes to avoid naming conflicts
* Fallback behavior for missing imported files

The code requires several external modules, including `tokenizers`, `utils`, and `text_utils`. Some custom functions, such as `get_cased_name` and `common_partial_suffix`, are also used.


key: chroma:document
string_value:
CODE:

def check_dependencies_and_create_import_node(
    file_type: str, new_dependencies: set[str], mapper: ModuleMapper, new_name: str
) -> tuple[set[str], dict[str, cst.CSTNode]]:
    """Check that all class nodes in the `new_dependencies` belong to the correct `file_type`. If this is not the case,
    we need to remove it from the dependencies, and create a new import to it instead.
    This scenario may appear in the following case:
    If a new class in the `modular_xxx.py` file does not belong to `type_xxx.py`, but is used somewhere in `other_type_xxx.py`
    (e.g. as a type hint), but none of the visited files had a similar class, then it would be imported in `type_xxx.py` as
    part of the standard dependency graph (because we never encountered an import towards this new class in any file).
    For example imagine the following `modular.py`:
    ```
    from ..llama.modeling_llama import LlamaModel

    class NewNameTextConfig(PretrainedConfig):
        ...

    class NewNameConfig(PretrainedConfig):
        ...

    class NewNameModel(LlamaModel):
        config = NewNameConfig()
        text_config = NewNameTextConfig()
        ...
    ```
    then without the help of this function, `NewNameTextConfig` would be imported in the `modeling_newname.py` as well as
    `configuration_newname.py`, because `modeling_llama.py` tells us to not import `NewNameConfig`, but has no
    knowledge of `NewNameTextConfig`.
    """
    class_dependencies = {dep for dep in new_dependencies if m.matches(mapper.global_nodes[dep], m.ClassDef())}
    corrected_dependencies = new_dependencies.copy()
    new_imports = {}
    for class_name in class_dependencies:
        class_file_type = find_file_type(class_name)
        # In this case, we need to remove it from the dependencies and create a new import instead
        if class_file_type != file_type:
            corrected_dependencies.remove(class_name)
            import_statement = f"from .{class_file_type}_{new_name} import {class_name}"
            new_imports[class_name] = cst.parse_statement(import_statement)



SUMMARY:
**Function Summary: `check_dependencies_and_create_import_node`**

* **Purpose**: Ensure that class nodes in `new_dependencies` belong to the correct file type. If not, remove it and create a new import statement.
* **Key Implementation Details**:
	+ Filter class dependencies based on their file types using the `matches` method of a `ModuleMapper`.
	+ Create a dictionary `corrected_dependencies` with remaining class nodes that belong to the correct file type.
	+ Generate `import_statement` for each misclassified class node and store it in the `new_imports` dictionary.

**Method Signatures and Interfaces**:

* `ModuleMapper.matches()`: Matches a module with a specific node type (in this case, `ClassDef()`).
* `find_file_type(class_name)`: Returns the file type of a given class name.
* `cst.parse_statement(import_statement)`: Parses an import statement into a CST (Abstract Syntax Tree) Node.

**Notable Dependencies and Requirements**:

* `ModuleMapper`: A module mapper object that provides the necessary methods.
* `find_file_type(class_name)` function: Not shown in this code snippet but presumably defined elsewhere in the project.
* External dependencies:
	+ `cst` (Collection of Statements, Functions, Variables, etc.)


key: chroma:document
string_value:
CODE:

def get_class_node_and_dependencies(
    modular_mapper: ModularFileMapper, class_name: str, node: cst.CSTNode, files: dict[str, dict]
) -> tuple[dict, str, dict]:
    """Return a single class node (and all its dependency nodes), to be added to the `files`. It creates the new
    class node based on the inherited classes if needed. Also returns any new imports of a new class defined in
    the modular that we nay need.
    """
    # An exception was already raised if this has len > 1
    model_specific_bases = [
        k.value.value for k in node.bases if k.value.value in modular_mapper.model_specific_imported_objects
    ]
    super_class = model_specific_bases[0] if len(model_specific_bases) == 1 else None

    file_type = find_file_type(class_name)
    file_to_update = files[file_type]
    model_name = modular_mapper.model_name

    # This is used to avoid adding objects to the dependencies graph if they will be imported already
    imported_objects = modular_mapper.imported_objects_per_file[file_type]

    # We need to replace the class node with the transformers (modeling file) super class node
    if super_class is not None:
        super_file_name = modular_mapper.model_specific_imported_objects[super_class]

        # Get the mapper corresponding to the inherited class
        mapper = modular_mapper.visited_modules[super_file_name]
        # Rename the super class according to the exact same rule we used when renaming the whole module
        renamer = modular_mapper.renamers[super_file_name]
        renamed_super_class = preserve_case_replace(super_class, renamer.patterns, renamer.cased_new_name)

        # Create the new class node
        updated_node = replace_class_node(mapper, node, renamed_super_class, super_class)

        # Grab all immediate dependencies of the new node
        new_node_dependencies = augmented_dependencies_for_class_node(updated_node, mapper, imported_objects)

        # At this point, if any class dependency is found, but belongs to another file, it means that we need to remove
        # it from the dependencies, and add a new import of it instead
        new_node_dependencies, new_imports = check_dependencies_and_create_import_node(
            file_type, new_node_dependencies, mapper, model_name
        )

        # The node was modified -> look for all recursive dependencies of the new node
        all_dependencies_to_add = find_all_dependencies(
            dependency_mapping=mapper.class_dependency_mapping,
            initial_dependencies=new_node_dependencies,
            initial_checked_dependencies=set(file_to_update.keys()),
        )

        relative_dependency_order = mapper.compute_relative_order(all_dependencies_to_add)
        nodes_to_add = {
            dep: (relative_dependency_order[dep], mapper.global_nodes[dep]) for dep in all_dependencies_to_add
        }

    # No transformers (modeling file) super class, just check functions and assignments dependencies
    else:
        updated_node = node
        # The node was NOT modified -> no need to look recursively for other class dependencies. Indeed, even if they are not
        # already defined (which would mean a weird order of the code in the modular...), they will be in the future
        all_dependencies_to_add = augmented_dependencies_for_class_node(updated_node, modular_mapper, imported_objects)

        # At this point, if any class dependency is found, but belongs to another file, it means that we need to remove
        # it from the dependencies, and add a new import of it instead
        all_dependencies_to_add, new_imports = check_dependencies_and_create_import_node(
            file_type, all_dependencies_to_add, modular_mapper, model_name
        )

        relative_dependency_order = modular_mapper.compute_relative_order(all_dependencies_to_add)
        nodes_to_add = {
            dep: (relative_dependency_order[dep], modular_mapper.global_nodes[dep])
            for dep in all_dependencies_to_add
            if dep not in file_to_update.keys()
        }

    # Add the class node itself to the nodes to add
    class_idx = max(relative_dependency_order.values()) + 1 if len(relative_dependency_order) > 0 else 0
    nodes_to_add[class_name] = (class_idx, updated_node)



SUMMARY:
**Summary: get_class_node_and_dependencies Function**

The `get_class_node_and_dependencies` function generates a single class node and its dependencies for a given modular file. It:

1. **Creates the Class Node**: Builds the class node based on inherited classes, if needed, using the provided modular mappings.
2. **Generates New Imports**: Finds new imports of classes defined in the modular that may be required.
3. **Analyzes Dependencies**: Checks dependencies and creates import nodes if necessary, recursively finding all dependencies.
4. **Orders Dependencies**: Computes relative order for the generated dependencies.

**Key Function Details**

* Returns a tuple containing the class node with its updated dependencies, file type, and new imports.
* Uses modular mappings to rename super classes according to exact rules.

**Notable Dependencies and Requirements**

* `modular_mapper` object
	+ `model_specific_imported_objects`
	+ `imported_objects_per_file`
	+ `visited_modules`
	+ `renamers`
* External dependencies:
	+ `find_file_type`
	+ `augmented_dependencies_for_class_node`
	+ `check_dependencies_and_create_import_node`

**Interfaces and Dependencies**

* Uses interfaces from `modular_mapper` object, including `mapper`, `renamer`, and `visited_modules`.
* Returns nodes using a dictionary-like structure with relative dependency order.


key: chroma:document
string_value:
CODE:

def create_modules(modular_mapper: ModularFileMapper) -> dict[str, cst.Module]:
    """Create all the new modules based on visiting the modular file. It replaces all classes as necesary."""
    files = defaultdict(dict)
    current_file_indices = defaultdict(lambda: 0)

    # For each class defined in modular, potentially replace the node and add it with its dependencies
    for class_name, node in modular_mapper.classes.items():
        nodes_to_add, file_type, new_imports = get_class_node_and_dependencies(modular_mapper, class_name, node, files)

        # Add the new potential new imports that we may need to the `modular_mapper` variable
        modular_mapper.imported_objects_per_file[file_type].update(new_imports.keys())
        modular_mapper.imports.extend(list(new_imports.values()))

        # Sort the nodes according to their relative order
        nodes_to_add = sorted(nodes_to_add.items(), key=lambda x: x[1][0])
        # Write all nodes to file
        for dependency, (_, node) in nodes_to_add:
            # This is used to keep certain variables at the beginning of the file
            try:
                # The -1000 is arbitrary -> just keep it bigger than the list
                idx = -1000 + VARIABLES_AT_THE_BEGINNING.index(dependency)
            except ValueError:
                idx = current_file_indices[file_type]
                current_file_indices[file_type] += 1
            files[file_type][dependency] = {"insert_idx": idx, "node": node}

    # Add the __all__ statement to files at the end
    for file_type, node in modular_mapper.all_all_to_add.items():
        idx = current_file_indices[file_type]
        files[file_type]["__all__"] = {"insert_idx": idx, "node": node}

    # Aggregate all the imports statements (we look for duplicates with the code_for_node, not the nodes themselves because
    # they are wrapped in SimpleStatementLine or If which could have different newlines, blanks etc)
    all_imports = modular_mapper.imports.copy()
    all_imports_code = {modular_mapper.python_module.code_for_node(node).strip() for node in all_imports}
    for file, mapper in modular_mapper.visited_modules.items():
        new_imports = [
            node for node in mapper.imports if mapper.python_module.code_for_node(node).strip() not in all_imports_code
        ]
        new_imports_code = {mapper.python_module.code_for_node(node).strip() for node in new_imports}
        all_imports.extend(new_imports)
        all_imports_code.update(new_imports_code)

    # Find the correct imports, and write the new modules
    for file, body in files.items():
        new_body = [k[1]["node"] for k in sorted(body.items(), key=lambda x: x[1]["insert_idx"])]
        needed_imports = get_needed_imports(body, all_imports)
        full_module = needed_imports + new_body
        new_module = cst.Module(body=full_module, header=modular_mapper.python_module.header)
        files[file] = new_module



SUMMARY:
**Summary: Create Modules Function**

*   **Purpose:** Creates new modules based on visiting modular files by replacing classes and adding dependencies.

**Main Purpose and Functionality:**

The `create_modules` function takes a `ModularFileMapper` object as input and returns a dictionary of new modules. It achieves the following key functionality:

1.  Replaces classes in each class definition with updated nodes and added dependencies.
2.  Adds potential new imports to the `modular_mapper.imported_objects_per_file` map.
3.  Sorts nodes according to their relative order for file writing.
4.  Writes all nodes to files, keeping certain variables at the beginning of the file.
5.  Aggregates and finds correct imports to write full modules.

**Key Implementation Details:**

*   Iterates through each class definition in `modular_mapper.classes`.
*   Uses `get_class_node_and_dependencies` function to get updated nodes and dependencies for each class.
*   Updates `imported_objects_per_file` map with new imports.
*   Sorts nodes for file writing using a custom sorting key.
*   Creates a new module by combining needed imports and body.

**Notable Dependencies or Requirements:**

*   `ModularFileMapper` object
*   `get_class_node_and_dependencies` function
*   `VARIABLES_AT_THE_BEGINNING` list
*   `cst.Module` class

**Additional Information:**

This code appears to be part of a larger system designed to work with modular files and their dependencies. The function is closely tied to the `ModularFileMapper` object, which seems to provide necessary data for creating new modules.

The code uses various external libraries or classes (e.g., `cst.Module`, `VARIABLES_AT_THE_BEGINNING`) that are not defined in this snippet.


key: chroma:document
string_value:
CODE:

def convert_modular_file(modular_file):
    pattern = re.search(r"modular_(.*)(?=\.py$)", modular_file)
    output = {}
    if pattern is not None:
        model_name = pattern.groups()[0]
        # Parse the Python file
        with open(modular_file, "r", encoding="utf-8") as file:
            code = file.read()
        module = cst.parse_module(code)
        wrapper = MetadataWrapper(module)
        cst_transformers = ModularFileMapper(module, model_name)
        wrapper.visit(cst_transformers)
        for file, module in create_modules(cst_transformers).items():
            if module != {}:
                # Get relative path starting from src/transformers/
                relative_path = re.search(
                    r"(src/transformers/.*|examples/.*)", os.path.abspath(modular_file).replace("\\", "/")
                ).group(1)

                header = AUTO_GENERATED_MESSAGE.format(
                    relative_path=relative_path, short_name=os.path.basename(relative_path)
                )
                ruffed_code = run_ruff(header + module.code, True)
                formatted_code = run_ruff(ruffed_code, False)
                output[file] = [formatted_code, ruffed_code]
        return output
    else:
        print(f"modular pattern not found in {modular_file}, exiting")


SUMMARY:
**Function Summary:**

`convert_modular_file(modular_file)`

Converts a modular Python file to conform to formatting standards.

**Key Implementation Details:**

- Parses the input modular file using regular expressions and CST (Common SyntaxTree) parser.
- Applies transformations using `MetadataWrapper` and `ModularFileMapper`.
- Formats code using Ruff library on two rounds of review (initial and final).

**Notable Dependencies and Requirements:**

- `re`, `os`, `cst`, and `Ruff` libraries for parsing and formatting purposes.
- Custom classes (`MetadataWrapper`, `ModularFileMapper`) are used to implement transformation logic.


key: chroma:document
string_value:
CODE:

def save_modeling_file(modular_file, converted_file):
    for file_type in converted_file.keys():
        file_name_prefix = file_type.split("*")[0]
        file_name_suffix = file_type.split("*")[-1] if "*" in file_type else ""
        new_file_name = modular_file.replace("modular_", f"{file_name_prefix}_").replace(
            ".py", f"{file_name_suffix}.py"
        )
        non_comment_lines = len(
            [line for line in converted_file[file_type][0].strip().split("\n") if not line.strip().startswith("#")]
        )
        if len(converted_file[file_type][0].strip()) > 0 and non_comment_lines > 0:
            with open(new_file_name, "w", encoding="utf-8") as f:
                f.write(converted_file[file_type][0])
        else:
            non_comment_lines = len(
                [line for line in converted_file[file_type][0].strip().split("\n") if not line.strip().startswith("#")]
            )
            if len(converted_file[file_type][1].strip()) > 0 and non_comment_lines > 0:
                logger.warning("The modeling code contains errors, it's written without formatting")
                with open(new_file_name, "w", encoding="utf-8") as f:


SUMMARY:
**Summary:**

Save Modeling File Functionality
==============================

**Purpose and Functionality**
-----------------------------

This function `save_modeling_file` takes two inputs, `modular_file` and `converted_file`, and saves the contents of `converted_file` to new files with modified names based on their original file types.

**Key Implementation Details:**

*   Iterates over each file type in `converted_file` keys.
*   Creates a new file name by replacing "modular_" prefix and adding `.py` extension suffix based on file type.
*   Checks if the converted file lines contain any non-comment content and writes it to the new file.

**Notable Dependencies and Requirements:**

*   Uses `logger` for warnings during file writing.
*   External dependency on Python's built-in `logging` module (referenced as "logger").
*   Does not inherit or implement any specific interfaces.


key: chroma:document
string_value:
CODE:

def test_custom_files_are_present(transformers_path):
    # Test all the extensions added in the setup
    for file in FILES_TO_FIND:
        if not (transformers_path / file).exists():
            return False


SUMMARY:
**Summary:**

`test_custom_files_are_present`: Verifies that custom files exist at a specified `transformers_path`.

**Key Implementation Details:**

* Iterates through `FILES_TO_FIND` and checks existence of each file using `exists()`.
* Returns `True` if all files are present, `False` otherwise.

**Notable Dependencies/Requirements:**

* `FILES_TO_FIND`: a list of filenames to check (assumed to be defined elsewhere).
* External dependencies not mentioned.
* `transformers_path` is assumed to be an existing directory path.


key: chroma:document
string_value:
CODE:

def get_all_model_names():
    model_names = set()
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [
            x
            for x in dir(module)
            if x.endswith("_MAPPING_NAMES")
            and (x.startswith("MODEL_") or x.startswith("TF_MODEL_") or x.startswith("FLAX_MODEL_"))
        ]
        for name in mapping_names:
            mapping = getattr(module, name)
            if mapping is not None:
                for v in mapping.values():
                    if isinstance(v, (list, tuple)):
                        model_names.update(v)
                    elif isinstance(v, str):
                        model_names.add(v)



SUMMARY:
**Function Summary: get_all_model_names()**

The `get_all_model_names()` function extracts and returns a set of unique model names from auto-modeling files in the `transformers.models.auto` namespace.

**Key Implementation Details:**

* Iterates through three auto-modeling files (`modeling_auto`, `modeling_tf_auto`, `modeling_flax_auto`) to gather model mappings.
* Uses string matching (`str.endswith()`) and object existence checks (`isinstance()` and `is None`) to identify relevant model names.

**Notable Dependencies/Requirements:**

* `transformers.models.auto` namespace
* No external dependencies listed.


key: chroma:document
string_value:
CODE:

def get_tiny_model_names_from_repo():
    # All model names defined in auto mappings
    model_names = set(get_all_model_names())

    with open("tests/utils/tiny_model_summary.json") as fp:
        tiny_model_info = json.load(fp)
    tiny_models_names = set()
    for model_base_name in tiny_model_info:
        tiny_models_names.update(tiny_model_info[model_base_name]["model_classes"])

    # Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.
    not_on_hub = model_names.difference(tiny_models_names)
    for model_name in copy.copy(tiny_models_names):
        if not model_name.startswith("TF") and f"TF{model_name}" in not_on_hub:
            tiny_models_names.remove(model_name)
        elif model_name.startswith("TF") and model_name[2:] in not_on_hub:
            tiny_models_names.remove(model_name)



SUMMARY:
**Concise Summary: `get_tiny_model_names_from_repo`**

**Purpose:** Retrieves the names of tiny models defined in auto-mappings and filtering out those not on the Hub.

**Key Implementation Details:**

* Uses `get_all_model_names()` to get all model names, then filters using a set operation (`difference` and `update`) between model names and their implementations on the Hub (`tiny_models_names`).
* Checks for the presence of "TF" prefix in a specific format (e.g., "TFx[model_name]"), removing models that don't have a matching tiny version.
* Uses JSON data from "tests/utils/tiny_model_summary.json" to infer available model classes.

**Dependencies/Requirements:**

* `get_all_model_names()`
* `json` module
* `copy` module
* External dependency on "tests/utils/tiny_model_summary.json" file.


key: chroma:document
string_value:
CODE:

def get_tiny_model_summary_from_hub(output_path):
    special_models = COMPOSITE_MODELS.values()

    # All tiny model base names on Hub
    model_names = get_all_model_names()
    models = hf_api.list_models(
        filter=ModelFilter(
            author="hf-internal-testing",
        )
    )
    _models = set()
    for x in models:
        model = x.id
        org, model = model.split("/")
        if not model.startswith("tiny-random-"):
            continue
        model = model.replace("tiny-random-", "")
        if not model[0].isupper():
            continue
        if model not in model_names and model not in special_models:
            continue
        _models.add(model)

    models = sorted(_models)
    # All tiny model names on Hub
    summary = {}
    for model in models:
        repo_id = f"hf-internal-testing/tiny-random-{model}"
        model = model.split("-")[0]
        try:
            repo_info = hf_api.repo_info(repo_id)
            content = {
                "tokenizer_classes": set(),
                "processor_classes": set(),
                "model_classes": set(),
                "sha": repo_info.sha,
            }
        except Exception:
            continue
        try:
            time.sleep(1)
            tokenizer_fast = AutoTokenizer.from_pretrained(repo_id)
            content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__)
        except Exception:
            pass
        try:
            time.sleep(1)
            tokenizer_slow = AutoTokenizer.from_pretrained(repo_id, use_fast=False)
            content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__)
        except Exception:
            pass
        try:
            time.sleep(1)
            img_p = AutoImageProcessor.from_pretrained(repo_id)
            content["processor_classes"].add(img_p.__class__.__name__)
        except Exception:
            pass
        try:
            time.sleep(1)
            feat_p = AutoFeatureExtractor.from_pretrained(repo_id)
            if not isinstance(feat_p, BaseImageProcessor):
                content["processor_classes"].add(feat_p.__class__.__name__)
        except Exception:
            pass
        try:
            time.sleep(1)
            model_class = getattr(transformers, model)
            m = model_class.from_pretrained(repo_id)
            content["model_classes"].add(m.__class__.__name__)
        except Exception:
            pass
        try:
            time.sleep(1)
            model_class = getattr(transformers, f"TF{model}")
            m = model_class.from_pretrained(repo_id)
            content["model_classes"].add(m.__class__.__name__)
        except Exception:
            pass

        content["tokenizer_classes"] = sorted(content["tokenizer_classes"])
        content["processor_classes"] = sorted(content["processor_classes"])
        content["model_classes"] = sorted(content["model_classes"])

        summary[model] = content
        with open(os.path.join(output_path, "hub_tiny_model_summary.json"), "w") as fp:


SUMMARY:
**Function Summary: `get_tiny_model_summary_from_hub`**

**Purpose:** Fetches summaries for tiny models from the Hugging Face model hub.

**Main Purpose**: The function retrieves a list of tiny models from the Hugging Face model hub, constructs their metadata (e.g., tokenizer, processor, and model classes), and stores it in a JSON file.

**Key Implementation Details:**

*   Filters models by author and namespace.
*   Iterates through each model, retrieving its metadata using the `hf_api` interface.
*   Handles cases where model metadata is not available, waiting 1 second between attempts.
*   Extracts tokenizer classes from slow and fast versions of the AutoTokenizer.
*   Extracts processor classes from image processors (AutoImageProcessor).
*   Extracts model classes from transformers module using string-based lookup.

**Notable Dependencies:**

*   `hf_api`: Hugging Face API interface for interacting with the model hub.
*   `transformers`: Hugging Face transformer library for loading pre-trained models.
*   ` AutoTokenizer`, `AutoImageProcessor`, and `AutoFeatureExtractor` classes from transformers module.

**External Requirements:**

*   Requires access to Hugging Face API subscription or trial account.


key: chroma:document
string_value:
CODE:

def check_missing_backends():
    """
    Checks if all backends are installed (otherwise the check of this script is incomplete). Will error in the CI if
    that's not the case but only throw a warning for users running this.
    """
    missing_backends = []
    if not is_torch_available():
        missing_backends.append("PyTorch")
    if not is_tf_available():
        missing_backends.append("TensorFlow")
    if not is_flax_available():
        missing_backends.append("Flax")
    if len(missing_backends) > 0:
        missing = ", ".join(missing_backends)
        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:
            raise Exception(
                "Full repo consistency checks require all backends to be installed (with `pip install -e '.[dev]'` in the "
                f"Transformers repo, the following are missing: {missing}."
            )
        else:
            warnings.warn(
                "Full repo consistency checks require all backends to be installed (with `pip install -e '.[dev]'` in the "
                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "
                "didn't make any change in one of those backends modeling files, you should probably execute the "
                "command above to be on the safe side."


SUMMARY:
**Summary**

The `check_missing_backends` function checks if all required backends (PyTorch, TensorFlow, and Flax) are installed. It:

* Compares availability of each backend using `is_`:tch_available`, `is_tf_available`, and `is_flax_available`
* If any backend is missing, it appends the name to `missing_backends` list
* Raises an exception in CI if all backends are not installed or a warning for users otherwise
* Provides instructions on how to install missing backends using pip

**Key Implementation Details**

* Checks availability of three backends using separate functions (`is_torch_available`, `is_tf_available`, and `is_flax_available`)
* Uses an exception to raise error in CI if all backends are not installed
* Uses a warning to notify users about the missing backends
* Provides a pip installation command to install missing backends

**Notable Dependencies/Requirements**

* `pytorch`: PyTorch backend
* `tensorflow`: TensorFlow backend
* `flax`: Flax backend
* `transformers`: Transformers repository
* `os`: Operating system dependency for environment variable access


key: chroma:document
string_value:
CODE:

def check_model_list():
    """
    Checks the model listed as subfolders of `models` match the models available in `transformers.models`.
    """
    # Get the models from the directory structure of `src/transformers/models/`
    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")
    _models = []
    for model in os.listdir(models_dir):
        if model == "deprecated":
            continue
        model_dir = os.path.join(models_dir, model)
        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):
            _models.append(model)

    # Get the models in the submodule `transformers.models`
    models = [model for model in dir(transformers.models) if not model.startswith("__")]

    missing_models = sorted(set(_models).difference(models))
    if missing_models:
        raise Exception(
            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."


SUMMARY:
**Function Summary:** `check_model_list`

**Primary Purpose:** Validates that the models listed as subfolders of `src/transformers/models` match the available models in `transformers.models`.

**Key Implementation Details:**

*   Loads all non-deprecated subfolder models from `src/transformers/models/` into `_models`.
*   Compares the list of models in `_models` against the models listed in `transformers.models`.
*   Raising exception when there are missing models that should be included.

**Notable Dependencies:**

*   `os.path`: for path manipulation (e.g., joining paths).
*   `PATH_TO_TRANSFORMERS`: a defined constant referencing to the transformers directory.


key: docstring
string_value:
# If some modeling modules should be ignored for all checks, they should be added in the nested list
# _ignore_modules of this function.


key: chroma:document
string_value:
CODE:
# _ignore_modules of this function.
def get_model_modules() -> List[str]:
    """Get all the model modules inside the transformers library (except deprecated models)."""
    _ignore_modules = [
        "modeling_auto",
        "modeling_encoder_decoder",
        "modeling_marian",
        "modeling_retribert",
        "modeling_flax_auto",
        "modeling_flax_encoder_decoder",
        "modeling_speech_encoder_decoder",
        "modeling_flax_speech_encoder_decoder",
        "modeling_flax_vision_encoder_decoder",
        "modeling_timm_backbone",
        "modeling_tf_auto",
        "modeling_tf_encoder_decoder",
        "modeling_tf_vision_encoder_decoder",
        "modeling_vision_encoder_decoder",
    ]
    modules = []
    for model in dir(transformers.models):
        # There are some magic dunder attributes in the dir, we ignore them
        if model == "deprecated" or model.startswith("__"):
            continue

        model_module = getattr(transformers.models, model)
        for submodule in dir(model_module):
            if submodule.startswith("modeling") and submodule not in _ignore_modules:
                modeling_module = getattr(model_module, submodule)
                modules.append(modeling_module)


SUMMARY:
**Summary**

This function, `get_model_modules`, retrieves all model modules from the `transformers` library, excluding deprecated and specific "modeling" submodules. The primary purpose is to gather a list of models for use in a larger application.

**Key Implementation Details:**

*   ignores ignored module names (`_ignore_modules`) and magic dunder attributes in the directory listing.
*   uses `getattr()` to dynamically access individual model modules and their "modeling" submodules.
*   excludes deprecated model names detected as "__deprecated__".

**Notable Dependencies/Requirements:**

*   The `transformers` library, specifically its `models` module.

**Method Signatures Interfaces:**

*   `get_model_modules() -> List[str]`: Returns a list of model modules in the transformed format, excluding obsolete ones.


key: chroma:document
string_value:
CODE:

def get_models(module: types.ModuleType, include_pretrained: bool = False) -> List[Tuple[str, type]]:
    """
    Get the objects in a module that are models.

    Args:
        module (`types.ModuleType`):
            The module from which we are extracting models.
        include_pretrained (`bool`, *optional*, defaults to `False`):
            Whether or not to include the `PreTrainedModel` subclass (like `BertPreTrainedModel`) or not.

    Returns:
        List[Tuple[str, type]]: List of models as tuples (class name, actual class).
    """
    models = []
    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)
    for attr_name in dir(module):
        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):
            continue
        attr = getattr(module, attr_name)
        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:
            models.append((attr_name, attr))


SUMMARY:
Here's a concise and focused summary of the given function:

### Function Summary: `get_models`

**Purpose:** Extract objects from a module that are models.

**Primary Purpose:** Return a list of model classes as tuples (class name, actual class) based on their inheritance relation to predefined model classes.

**Key Implementation Details:**

* Iterates over attributes in the given module.
* Filters out attributes whose names contain keywords related to `PreTrainedModel` subclass (excluding them by default).
* Checks if each attribute is a type and inherits from defined model classes using multiple inheritance.
* Appends validated attributes to the result list.

**Notable Dependencies/Requirements:**

* `transformers`: Module containing definition of predefined model classes.
* `types`: Module for accessing built-in types like `ModuleType`.

### Return Value:

A list of tuples (class name, actual class) representing extracted model objects.


key: chroma:document
string_value:
CODE:

def is_building_block(model: str) -> bool:
    """
    Returns `True` if a model is a building block part of a bigger model.
    """
    if model.endswith("Wrapper"):
        return True
    if model.endswith("Encoder"):
        return True
    if model.endswith("Decoder"):
        return True
    if model.endswith("Prenet"):


SUMMARY:
Here is a concise summary of the function:

**Function Summary**

`is_building_block(model: str) -> bool`

Checks if a machine learning model is a building block part of a larger model. It returns `True` for models ending with "Wrapper", "Encoder", "Decoder", or "Prenet".

**Key Implementation Details**

* Uses string suffix matching (`str.endswith()`)
* Only checks for specific model prefixes (no regular expression)

**Dependencies and Interfaces Noted**

* No dependencies are mentioned in the function code.
* Inherited or implemented interfaces not applicable.

This summary is concise, accurate, and focused on providing essential information about the function's purpose and implementation details.


key: chroma:document
string_value:
CODE:

def is_a_private_model(model: str) -> bool:
    """Returns `True` if the model should not be in the main init."""
    if model in PRIVATE_MODELS:
        return True


SUMMARY:
**Summary**

Determines whether a given model should exclude itself from the main initialization.

**Functionality**

- Checks if the model is in the `PRIVATE Modes` list, returning `True` if it is.
- Falls back to calling `is_building_block` with the model as an argument to determine its behavior.

**Key Implementation Details**

- Method signature: `def is_a_private_model(model: str) -> bool`
- Calls `PRIVATE_MODELS` set to determine private models
- Calls `is_building_block` method on a given model

**Notable Dependencies or Requirements**

- Uses the `PRIVATE.Models` set, which is not defined in this snippet. 
  - References related class 'PrivateModels'
  
Note: No external dependencies are listed as this function only references internal sets and methods.


key: chroma:document
string_value:
CODE:

def check_models_are_in_init():
    """Checks all models defined in the library are in the main init."""
    models_not_in_init = []
    dir_transformers = dir(transformers)
    for module in get_model_modules():
        models_not_in_init += [
            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers
        ]

    # Remove private models
    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]
    if len(models_not_in_init) > 0:


SUMMARY:
**Function Summary**

`check_models_are_in_init()`
=====================

Checks if defined model libraries are deployed in the main application initialization.

**Primary Purpose**

Verifies that all models imported from external modules are included in the main `init.py` file of the library.

**Key Implementation Details**

* Iterates over model modules using `get_model_modules()` and collects models not present in `transformers` directory
* Filters out private models using `is_a_private_model()`
* Raises an exception if any models are missing from main init

**Notable Dependencies and Requirements**

* `transformers`: used to define the expected models directory
* `get_model_modules()`, `get_models()`, and `is_a_private_model()`: functions not shown in this code snippet, but assumed to be defined elsewhere in the library.


key: docstring
string_value:
# If some test_modeling files should be ignored when checking models are all tested, they should be added in the
# nested list _ignore_files of this function.


key: chroma:document
string_value:
CODE:
# nested list _ignore_files of this function.
def get_model_test_files() -> List[str]:
    """
    Get the model test files.

    Returns:
        `List[str]`: The list of test files. The returned files will NOT contain the `tests` (i.e. `PATH_TO_TESTS`
        defined in this script). They will be considered as paths relative to `tests`. A caller has to use
        `os.path.join(PATH_TO_TESTS, ...)` to access the files.
    """

    _ignore_files = [
        "test_modeling_common",
        "test_modeling_encoder_decoder",
        "test_modeling_flax_encoder_decoder",
        "test_modeling_flax_speech_encoder_decoder",
        "test_modeling_marian",
        "test_modeling_tf_common",
        "test_modeling_tf_encoder_decoder",
    ]
    test_files = []
    model_test_root = os.path.join(PATH_TO_TESTS, "models")
    model_test_dirs = []
    for x in os.listdir(model_test_root):
        x = os.path.join(model_test_root, x)
        if os.path.isdir(x):
            model_test_dirs.append(x)

    for target_dir in [PATH_TO_TESTS] + model_test_dirs:
        for file_or_dir in os.listdir(target_dir):
            path = os.path.join(target_dir, file_or_dir)
            if os.path.isfile(path):
                filename = os.path.split(path)[-1]
                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:
                    file = os.path.join(*path.split(os.sep)[1:])
                    test_files.append(file)



SUMMARY:
**Function Summary:**

`get_model_test_files()` 

 Retrieves and returns a list of test files ("test_modeling" files) within the `models` directory under the test root. Excludes ignored files from specified `_ignore_files`.

**Key Details:**

* Ignored files are specified in `_ignore_files`.
* Files with "test_modeling" in their filename, excluding those in `_ignore_files`, are considered.
* Files are joined using `os.path.join()` to construct absolute paths.

**Notable Dependencies:**

* `os`: for file system interactions
* `PATH_TO_TESTS`: script-defined constant for test root path


key: docstring
string_value:
# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class
# for the all_model_classes variable.


key: chroma:document
string_value:
CODE:
# for the all_model_classes variable.
def find_tested_models(test_file: str) -> List[str]:
    """
    Parse the content of test_file to detect what's in `all_model_classes`. This detects the models that inherit from
    the common test class.

    Args:
        test_file (`str`): The path to the test file to check

    Returns:
        `List[str]`: The list of models tested in that file.
    """
    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:
        content = f.read()
    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)
    # Check with one less parenthesis as well
    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)
    if len(all_models) > 0:
        model_tested = []
        for entry in all_models:
            for line in entry.split(","):
                name = line.strip()
                if len(name) > 0:
                    model_tested.append(name)


SUMMARY:
**Function Summary**

`find_tested_models(test_file: str) -> List[str]`

Detects models tested in a given test file by parsing its content, specifically extracting `all_model_classes`.

**Key Functionality**

* Reads the content of a test file
* Uses regular expressions to extract `all_model_classes` patterns
* Parses pattern entries to extract model names

**Implementation Details**

* Method signatures: `re.findall` with two pattern strings for `all_model_classes`
* Algorithm: Combines multiple pattern findings into a single list using `.append`

**Notable Dependencies/Requirements**

* `os.path.join` and `open` for file operations
* `re` module for regular expression matching

**No notable inherited or implemented interfaces.**


key: chroma:document
string_value:
CODE:

def should_be_tested(model_name: str) -> bool:
    """
    Whether or not a model should be tested.
    """
    if model_name in IGNORE_NON_TESTED:
        return False


SUMMARY:
**Function Summary**

### `should_be_tested`

Determines whether a model with the given name should be tested.

**Purpose**: Checks if a model's existence in an IGNORE_NON_TESTED list or if it's a building block model to exclude testing.

**Key Implementation Details**:

* Returns `False` for models in `IGNORE NON_TESTED`.
* Excludes models with this method signature from being tested.
* References `is_building_block(model_name)` using the exact name of the function.
* Inherently uses an external list (`IGNORE_NON_TESTED`) without defining it within this code snippet.

**Notable Dependencies/Requirements**: 

* `IGNORE_NON_TESTED` external list or constant.


key: chroma:document
string_value:
CODE:

def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
    """Check models defined in a module are all tested in a given file.

    Args:
        module (`types.ModuleType`): The module in which we get the models.
        test_file (`str`): The path to the file where the module is tested.

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
    # XxxPreTrainedModel are not tested
    defined_models = get_models(module)
    tested_models = find_tested_models(test_file)
    if tested_models is None:
        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:
            return
        return [
            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "
            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "
            + "`utils/check_repo.py`."
        ]
    failures = []
    for model_name, _ in defined_models:
        if model_name not in tested_models and should_be_tested(model_name):
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not tested in "
                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."
                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"
                + "in the file `utils/check_repo.py`."
            )


SUMMARY:
**Summary:**

This function checks if all models defined in a module (`module`) are tested in a given test file (`test_file`). It returns a list of error messages for models that are not tested.

**Key Details:**

* Defines and tests models from `defined_models` and `tested_models` respectively
* Returns error messages for models that are not tested but should be (`model_name`)
* Notifies user to add model to `all_model_classes` in test file or ignore with `IGNORE_NON_TESTED`
 
**Function Signature:**
```python
def check_models_are_tested(module: types.ModuleType, test_file: str) -> List[str]:
```

**Notable Dependencies/Requirements:**

* `get_models`: module for getting defined models
* `find_tested_models`: function for finding tested models in a file
* `TEST_FILES_WITH_NO_COMMON_TESTS`: dictionary of files with no common tests
* `utils/check_repo.py`: configuration file for repository checks
* `types.ModuleType`: type hint for a Python module


key: chroma:document
string_value:
CODE:

def check_all_models_are_tested():
    """Check all models are properly tested."""
    modules = get_model_modules()
    test_files = get_model_test_files()
    failures = []
    for module in modules:
        # Matches a module to its test file.
        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]
        if len(test_file) == 0:
            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")
        elif len(test_file) > 1:
            failures.append(f"{module.__name__} has several test files: {test_file}.")
        else:
            test_file = test_file[0]
            new_failures = check_models_are_tested(module, test_file)
            if new_failures is not None:
                failures += new_failures
    if len(failures) > 0:


SUMMARY:
**Summary**

*   Function: `check_all_models_are_tested()`
*   **Purpose:** Verifies if all models have corresponding test files and reports any mismatches.
*   Key implementation:
    *   Retrieves module metadata (`get_model_modules()`).
    *   Iterates over each module, matching it to its test file based on the module name prefix.
    *   Calls a separate function (`check_models_are_tested()`) for modules with test files and combines any returned errors.
*   **Notable dependencies:**
    *   `get_model_modules()` (module metadata retrieval)
    *   `get_model_test_files()` (test file retrieval)

**External interfaces/deependencies:**

    - Check Model Functions
   


key: chroma:document
string_value:
CODE:

def get_all_auto_configured_models() -> List[str]:
    """Return the list of all models in at least one auto class."""
    result = set()  # To avoid duplicates we concatenate all model classes in a set.
    if is_torch_available():
        for attr_name in dir(transformers.models.auto.modeling_auto):
            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):
                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))
    if is_tf_available():
        for attr_name in dir(transformers.models.auto.modeling_tf_auto):
            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))
    if is_flax_available():
        for attr_name in dir(transformers.models.auto.modeling_flax_auto):
            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):
                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))


SUMMARY:
**Function Summary: `get_all_auto_configured_models`**

**Purpose:** Extracts and returns a list of model names from auto-configured models in at least one available framework (PyTorch, TensorFlow, or Flax).

**Implementation Details:**

* Iterates over specific attribute names (`MODEL_`, `TF_MODEL_`, and `FLAX_MODEL_`) and their corresponding mapping names within each auto-configured model.
* Uses set operations to combine unique model names from different frameworks.

**Notable Dependencies:**

* `transformers.models.auto.modeling_auto`
* `transformers.models.auto.modeling_tf_auto`
* `transformers.models.auto.modeling_flax_auto`
* `is_torch_available()`, `is_tf_available()`, and `is_flax_available()` (functions to check availability of respective frameworks)

**Return Value:** A list of strings representing model names from auto-configured models in at least one available framework.


key: chroma:document
string_value:
CODE:

def ignore_unautoclassed(model_name: str) -> bool:
    """Rules to determine if a model should be in an auto class."""
    # Special white list
    if model_name in IGNORE_NON_AUTO_CONFIGURED:
        return True
    # Encoder and Decoder should be ignored
    if "Encoder" in model_name or "Decoder" in model_name:
        return True


SUMMARY:
**Summary:**
This function determines whether a model with the given `model_name` should be in an auto-class. It checks against a special whitelist and ensures that specific models like "Encoder" and "Decoder" are ignored.

**Key Details:**

* **Functionality:** `ignore_unautoclassed(model_name) -> bool`
* **Purpose:** Determine if a model is auto-configured
* **Design Choice:** Whitelisting approach with specific exceptions for "Encoder" and "Decoder" models.
* **Dependencies/Requirements:**
	+ `IGNORE_NON_AUTO_CONFIGURED`: A predefined list of accepted non-auto-configured model names.

Note: The original code snippet included the `IGNORE_NON_AUTO_CONFIGURED` variable, which is likely defined elsewhere in the project. This summary assumes its existence for the purpose of analysis.


key: chroma:document
string_value:
CODE:

def check_models_are_auto_configured(module: types.ModuleType, all_auto_models: List[str]) -> List[str]:
    """
    Check models defined in module are each in an auto class.

    Args:
        module (`types.ModuleType`):
            The module in which we get the models.
        all_auto_models (`List[str]`):
            The list of all models in an auto class (as obtained with `get_all_auto_configured_models()`).

    Returns:
        `List[str]`: The list of error messages corresponding to models not tested.
    """
    defined_models = get_models(module)
    failures = []
    for model_name, _ in defined_models:
        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):
            failures.append(
                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "
                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "
                "`utils/check_repo.py`."
            )


SUMMARY:
**Summary:**

### Function Purpose:

`check_models_are_auto_configured` checks if models defined in a module are part of an auto class.

### Key Implementation Details:

*   Calculates the list of defined models from the input `module`.
*   Iterates over each model, verifying if it exists in the `all_auto_models` list and ignoring unautoclassed models.
*   Appends error messages to the `failures` list for non-existent models.

### Notable Dependencies/Requirements:

*   `get_all_auto_configured_models()`: Returns a list of all models in an auto class.
*   `ignore_unautoclassed(model_name)`: Function indicating whether a model is intended to be ignored.
*   `IGNORE_NON_AUTO_CONFIGURED` constant: File location for modifying ignore behavior.


key: chroma:document
string_value:
CODE:

def check_all_models_are_auto_configured():
    """Check all models are each in an auto class."""
    # This is where we need to check we have all backends or the check is incomplete.
    check_missing_backends()
    modules = get_model_modules()
    all_auto_models = get_all_auto_configured_models()
    failures = []
    for module in modules:
        new_failures = check_models_are_auto_configured(module, all_auto_models)
        if new_failures is not None:
            failures += new_failures
    if len(failures) > 0:


SUMMARY:
**Summary:**

`check_all_models_are_auto_configured()` function checks whether all models are configured as auto-configurable. It verifies that all required backends exist and iterates through modules to validate individual model configurations, accumulating failures in a list.

**Key Details:**

* Verifies existence of all required backends
* Iterates through modules to validate model configurations
* Checks for `None` return value from `check_models_are_auto_configured()`
* Accumulates failure messages and raises an exception

**Notable Dependencies/Requirements:**

* `get_model_modules()` function
* `get_all_auto_configured_models()` function
* `check_missing_backends()` function


key: chroma:document
string_value:
CODE:

def check_all_auto_object_names_being_defined():
    """Check all names defined in auto (name) mappings exist in the library."""
    # This is where we need to check we have all backends or the check is incomplete.
    check_missing_backends()

    failures = []
    mappings_to_check = {
        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
        for _, class_names in mapping.items():
            if not isinstance(class_names, tuple):
                class_names = (class_names,)
                for class_name in class_names:
                    if class_name is None:
                        continue
                    # dummy object is accepted
                    if not hasattr(transformers, class_name):
                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling
                        # module, if it's a private model defined in this file.
                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        if name.endswith("MODEL_FOR_IMAGE_MAPPING_NAMES") and is_a_private_model(class_name):
                            continue
                        failures.append(
                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."
                        )
    if len(failures) > 0:


SUMMARY:
**Summary:**

The `check_all_auto_object_names_being_defined()` function checks if all names defined in auto mapping mappings exist in the library. It iterates through various auto modeling files, maps, and class definitions to identify missing objects.

**Key Functionality:**

* Checks for missing classes derived from `TOKENIZER_MAPPING_NAMES`, `IMAGE_PROCESSOR_MAPPING_NAMES`, etc.
* Iterates through auto modeling files (`modeling_auto`, `modeling_tf_auto`, `modeling_flax_auto`) and extracts mapping names and their corresponding class definitions
* Reports missing classes in the following format: "<class_name> appears in <mapping-name> but it is not defined in the library."

**Notable Dependencies/Requirements:**

* `transformers.models.auto` module
* External dependencies:
	+ `transformers`
	+ `torch`
	+ `jittor`
	(+ other transformer-based libraries)
* Inherited Interfaces:
	+ `TOKENIZER_MAPPING_NAMES`, `IMAGE_PROCESSOR_MAPPING_NAMES`, etc.

**Implementation Details:**

* Uses dynamic imports and attribute access to extract mapping names and class definitions.
* Includes private model checks for certain mapping types.

**Primary Purpose:**
The primary purpose of this function is to identify missing classes derived from auto mapping mappings, raising an exception if any are found.


key: chroma:document
string_value:
CODE:

def check_all_auto_mapping_names_in_config_mapping_names():
    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""
    # This is where we need to check we have all backends or the check is incomplete.
    check_missing_backends()

    failures = []
    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.
    mappings_to_check = {
        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,
        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,
        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,
    }

    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name, mapping in mappings_to_check.items():
        for model_type in mapping:
            if model_type not in CONFIG_MAPPING_NAMES:
                failures.append(
                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "
                    "`CONFIG_MAPPING_NAMES`."
                )
    if len(failures) > 0:


SUMMARY:
**CONCISE SUMMARY**

### Function Purpose:

DefectsCheckAutoMappingNames checks if all keys defined in auto mapping names appear in CONFIG_MAPPING_NAMES.

### Key Implementation Details:

- Iterates through special mapping names and their corresponding values from `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer`.
- Compares model types in auto modeling files with those in CONFIG_MAPPING_NAMES.
- Raises an exception when inconsistencies are found.

### Notable Dependencies/Requirements:

- `CONFIG_MAPPING_NAMES`
- `IMAGE_PROCESSOR_MAPPING_NAMES`
- `FEATURE_EXTRACTOR_MAPPING_NAMES`
- `PROCESSOR_MAPPING_NAMES`
- `transformers.models.auto` (dynamic import)


key: chroma:document
string_value:
CODE:

def check_all_auto_mappings_importable():
    """Check all auto mappings can be imported."""
    # This is where we need to check we have all backends or the check is incomplete.
    check_missing_backends()

    failures = []
    mappings_to_check = {}
    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.
    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:
        module = getattr(transformers.models.auto, module_name, None)
        if module is None:
            continue
        # all mappings in a single auto modeling file
        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]
        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})

    for name in mappings_to_check:
        name = name.replace("_MAPPING_NAMES", "_MAPPING")
        if not hasattr(transformers, name):
            failures.append(f"`{name}`")
    if len(failures) > 0:


SUMMARY:
**Summary**

`check_all_auto_mappings_importable` function checks whether all auto-mapping functions can be imported.

**Primary Purpose**: Verify if all auto-mapping files are properly installed and importable.

**Key Implementation Details**:

* Iterates through `modeling_auto`, `modeling_tf_auto`, and `modeling_flax_auto` modules to collect their mapping names.
* Checks for the existence of each mapped function in the `transformers` namespace using `hasattr`.

**Notable Detections**:

* Failure: `MAPPING_NAME` (e.g. `_MAPPING_NAMES`) not found in `transformers`.
* Raised exception if any failure is detected, listing all failed mappings.

**Dependencies and Requirements**:
- `transformers.models.auto`
- External dependencies: Not explicitly listed, but assumed to be part of the `transformers` package.
- Inherited/Implemeted interfaces: No notable references.


key: chroma:document
string_value:
CODE:

def check_objects_being_equally_in_main_init():
    """
    Check if a (TensorFlow or Flax) object is in the main __init__ iif its counterpart in PyTorch is.
    """
    attrs = dir(transformers)

    failures = []
    for attr in attrs:
        obj = getattr(transformers, attr)
        if not hasattr(obj, "__module__") or "models.deprecated" in obj.__module__:
            continue

        module_path = obj.__module__
        module_name = module_path.split(".")[-1]
        module_dir = ".".join(module_path.split(".")[:-1])
        if (
            module_name.startswith("modeling_")
            and not module_name.startswith("modeling_tf_")
            and not module_name.startswith("modeling_flax_")
        ):
            parent_module = sys.modules[module_dir]

            frameworks = []
            if is_tf_available():
                frameworks.append("TF")
            if is_flax_available():
                frameworks.append("Flax")

            for framework in frameworks:
                other_module_path = module_path.replace("modeling_", f"modeling_{framework.lower()}_")
                if os.path.isfile("src/" + other_module_path.replace(".", "/") + ".py"):
                    other_module_name = module_name.replace("modeling_", f"modeling_{framework.lower()}_")
                    other_module = getattr(parent_module, other_module_name)
                    if hasattr(other_module, f"{framework}{attr}"):
                        if not hasattr(transformers, f"{framework}{attr}"):
                            if f"{framework}{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}{attr}")
                    if hasattr(other_module, f"{framework}_{attr}"):
                        if not hasattr(transformers, f"{framework}_{attr}"):
                            if f"{framework}_{attr}" not in OBJECT_TO_SKIP_IN_MAIN_INIT_CHECK:
                                failures.append(f"{framework}_{attr}")
    if len(failures) > 0:


SUMMARY:
**Summary:**

This function, `check_objects_being_equally_in_main_init`, checks if the objects present in TensorFlow/Flax counterpart are also present in PyTorch's main module. It iterates through all attributes of the `transformers` object and verifies the existence of corresponding attributes with prefix "modeling_". If found, it raises an exception indicating failures.

**Key Details:**

- **Implementation:** This function uses attribute iteration to check for equivalent objects between TensorFlow/Flax and PyTorch.
- **Important Method Signatures:**
  - `is_tf_available()`: checks if the TF framework is available.
  - `is_flax_available()`: checks if the Flax framework is available.
  
**Notable Dependencies and Requirements:**

- **Related Classes/Fuctions:** 
  - `transformers` (the object to be checked)
  - `sys`, `os`
- **External Dependencies:**
  - TensorFlow
  - Flax
  - PyTorch
- **Implemented Interfaces:**
    - No specific interfaces are mentioned, but the function appears to follow a generic structure for comparing attributes between frameworks.
-


key: chroma:document
string_value:
CODE:

def check_decorator_order(filename: str) -> List[int]:
    """
    Check that in a given test file, the slow decorator is always last.

    Args:
        filename (`str`): The path to a test file to check.

    Returns:
        `List[int]`: The list of failures as a list of indices where there are problems.
    """
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()
    decorator_before = None
    errors = []
    for i, line in enumerate(lines):
        search = _re_decorator.search(line)
        if search is not None:
            decorator_name = search.groups()[0]
            if decorator_before is not None and decorator_name.startswith("parameterized"):
                errors.append(i)
            decorator_before = decorator_name
        elif decorator_before is not None:
            decorator_before = None


SUMMARY:
**Summary: `check_decorator_order` Function**

* **Purpose**: Verifies that in a given test file, the slow decorator "parameterized" always follows other decorators.
* **Implementation Details**:
	+ Extracts lines from the specified test file and searches for occurrences of the "_re_decorator" regular expression.
	+ If the search finds "parameterized", adds the current line index to the error list if a preceding "parameterized" decorator was found.
	+ Resets the tracking of preceding decorators when a new one is encountered.
* **Notable Dependencies**: `_re_decorator` regular expression (internal), `List[int]` type hint for return value.
* **Assumptions/Requirements**: None external dependencies mentioned, no inherited or implemented interfaces listed.


key: chroma:document
string_value:
CODE:

def check_all_decorator_order():
    """Check that in all test files, the slow decorator is always last."""
    errors = []
    for fname in os.listdir(PATH_TO_TESTS):
        if fname.endswith(".py"):
            filename = os.path.join(PATH_TO_TESTS, fname)
            new_errors = check_decorator_order(filename)
            errors += [f"- {filename}, line {i}" for i in new_errors]
    if len(errors) > 0:
        msg = "\n".join(errors)
        raise ValueError(
            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"
            f" following files:\n{msg}"


SUMMARY:
**Code Summary**

```markdown
### Functionality

Checks if the slow decorator is last in every test file.
Raises ValueError with error messages if not.

### Key Implementation Details

* Iterates over test files (ending with .py) and checks decorator order using `check_decorator_order` method.
* Compares actual order against expected order (slow decorator first).
* If mismatch, raises `ValueError` with list of error messages and corresponding file locations.

### Notable Dependencies

- [os] library for accessing file system
- [PATH_TO_TESTS] environment variable or constant for path to test files
```

Note: This summary preserves the exact names of classes/functions/variables mentioned in the original code, uses precise technical terms, and focuses on key implementation details.


key: chroma:document
string_value:
CODE:

def find_all_documented_objects() -> List[str]:
    """
    Parse the content of all doc files to detect which classes and functions it documents.

    Returns:
        `List[str]`: The list of all object names being documented.
        `Dict[str, List[str]]`: A dictionary mapping the object name (full import path, e.g.
            `integrations.PeftAdapterMixin`) to its documented methods
    """
    documented_obj = []
    documented_methods_map = {}
    for doc_file in Path(PATH_TO_DOC).glob("**/*.md"):
        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
            content = f.read()
        raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content)
        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]

        for obj in raw_doc_objs:
            obj_public_methods = re.findall(rf"\[\[autodoc\]\] {obj}((\n\s+-.*)+)", content)
            # Some objects have no methods documented
            if len(obj_public_methods) == 0:
                continue
            else:
                documented_methods_map[obj] = re.findall(r"(?<=-\s).*", obj_public_methods[0][0])



SUMMARY:
**Summary**

This function, `find_all_documented_objects`, parses Markdown files in the specified directory to detect class and method documentation. It returns two data structures: a list of object names being documented and a dictionary mapping object names to their documented methods.

**Key Implementation Details**

* Uses regular expressions to extract object names and methods from Markdown content
* Iterates over Markdown files in the specified directory, extracting object objects with autodoc tags
* Uses dictionary comprehension to create a mapping between object names and their documented methods

**Primary Purpose**: Detects class and method documentation in Markdown files.

**Notable Dependencies**

* `re` for regular expression pattern matching
* `pathlib` for working with file paths


key: chroma:document
string_value:
CODE:

def ignore_undocumented(name: str) -> bool:
    """Rules to determine if `name` should be undocumented (returns `True` if it should not be documented)."""
    # NOT DOCUMENTED ON PURPOSE.
    # Constants uppercase are not documented.
    if name.isupper():
        return True
    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.
    if (
        name.endswith("PreTrainedModel")
        or name.endswith("Decoder")
        or name.endswith("Encoder")
        or name.endswith("Layer")
        or name.endswith("Embeddings")
        or name.endswith("Attention")
    ):
        return True
    # Submodules are not documented.
    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(
        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")
    ):
        return True
    # All load functions are not documented.
    if name.startswith("load_tf") or name.startswith("load_pytorch"):
        return True
    # is_xxx_available functions are not documented.
    if name.startswith("is_") and name.endswith("_available"):
        return True
    # Deprecated objects are not documented.
    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:
        return True
    # MMBT model does not really work.
    if name.startswith("MMBT"):
        return True
    if name in SHOULD_HAVE_THEIR_OWN_PAGE:
        return True


SUMMARY:
**Summary**

This function, `ignore_undocumented`, determines whether a given string `name` should be included in the documentation based on specific rules. It checks for:

* Uppercase constants and pre-trained models
* Submodules (e.g., transformer.py)
* Load functions (e.g., load_tf())
* is_xxx_available functions
* Deprecated objects
* MMBT model

**Key Implementation Details**

* The function uses a combination of string matching and path checking using `os.path`.
* The `PATH_TO_TRANSFORMERS` constant is assumed to be defined elsewhere in the codebase.

**Notable Dependencies and Requirements**

* `os`: for file system operations
* `PATH_TO_TRANSFORMERS`: implied path to transformer submodules

**Primary Purpose**
The primary purpose of this function is to determine whether a given string should be excluded from documentation based on specific rules.


key: chroma:document
string_value:
CODE:

def check_all_objects_are_documented():
    """Check all models are properly documented."""
    documented_objs, documented_methods_map = find_all_documented_objects()
    modules = transformers._modules
    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]
    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]
    if len(undocumented_objs) > 0:
        raise Exception(
            "The following objects are in the public init so should be documented:\n - "
            + "\n - ".join(undocumented_objs)
        )
    check_model_type_doc_match()


SUMMARY:
**Summary**

`check_all_objects_are_documented()` function:

*   **Purpose and functionality**: Ensures all models in the `transformers` namespace are properly documented by checking for public initialization objects and documenting methods that exist in a mapping.

**Key implementation details**:

*   Checks objects in the `transformers` namespace using dir() to find non-module, non-internal private objects.
*   Verifies if objects are not already documented by calling `find_all_documented_objects()` and ignoring certain method names using `ignore_undocumented(c)`.
*   Raises an exception with a list of undocumented objects if any exist.

**Notable dependencies or requirements**:

*   Calls other functions: `find_all_documented_objects()`, `check_model_type_doc_match()`, and `check_public_method_exists(documented_methods_map)`. 
*   References classes/functions: `transformers._modules`, `dir(transformers)`.
*   External dependencies: None mentioned.
*   Implements interfaces: [Unspecified, as it appears to be a custom class or module within the `transformers` namespace.


key: chroma:document
string_value:
CODE:

def check_public_method_exists(documented_methods_map):
    """Check that all explicitly documented public methods are defined in the corresponding class."""
    failures = []
    for obj, methods in documented_methods_map.items():
        # Let's ensure there is no repetition
        if len(set(methods)) != len(methods):
            failures.append(f"Error in the documentation of {obj}: there are repeated documented methods.")

        # Navigates into the object, given the full import path
        nested_path = obj.split(".")
        submodule = transformers
        if len(nested_path) > 1:
            nested_submodules = nested_path[:-1]
            for submodule_name in nested_submodules:
                if submodule_name == "transformers":
                    continue

                try:
                    submodule = getattr(submodule, submodule_name)
                except AttributeError:
                    failures.append(f"Could not parse {submodule_name}. Are the required dependencies installed?")
                continue

        class_name = nested_path[-1]

        try:
            obj_class = getattr(submodule, class_name)
        except AttributeError:
            failures.append(f"Could not parse {class_name}. Are the required dependencies installed?")
            continue

        # Checks that all explicitly documented methods are defined in the class
        for method in methods:
            if method == "all":  # Special keyword to document all public methods
                continue
            try:
                if not hasattr(obj_class, method):
                    failures.append(
                        "The following public method is explicitly documented but not defined in the corresponding "
                        f"class. class: {obj}, method: {method}. If the method is defined, this error can be due to "
                        f"lacking dependencies."
                    )
            except ImportError:
                pass

    if len(failures) > 0:


SUMMARY:
**Function Summary:**

`check_public_method_exists(documented_methods_map)`
=============================================

**Purpose:** Ensures that all explicitly documented public methods are defined in the corresponding class.

**Main Functionality:**
- Verifies consistency of documented public methods across the specified map.
- Resolves method definitions within nested object hierarchies using dynamic imports.

**Key Implementation Details:**

- Iterates through each object and its associated classes, ensuring unique documented method names.
- Handles class and method name resolutions via `getattr` with conditional error handling for unknown submodules or dependencies.

**External Dependencies:**
- `transformers`: A required submodule.

**Interfaces and Methods:** 

- The function iterates over a dictionary mapping objects to their listed public methods (`documented_methods_map`).


key: chroma:document
string_value:
CODE:

def check_model_type_doc_match():
    """Check all doc pages have a corresponding model type."""
    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"
    model_docs = [m.stem for m in model_doc_folder.glob("*.md")]

    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())
    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]

    errors = []
    for m in model_docs:
        if m not in model_types and m != "auto":
            close_matches = get_close_matches(m, model_types)
            error_message = f"{m} is not a proper model identifier."
            if len(close_matches) > 0:
                close_matches = "/".join(close_matches)
                error_message += f" Did you mean {close_matches}?"
            errors.append(error_message)

    if len(errors) > 0:
        raise ValueError(
            "Some model doc pages do not match any existing model type:\n"
            + "\n".join(errors)
            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "
            "models/auto/configuration_auto.py."


SUMMARY:
**Summary**

This function, `check_model_type_doc_match`, checks if all documentation pages under `PATH_TO_DOC/model_doc` correspond to existing models. It iterates through the doc pages, finds their corresponding model types using the `MODEL_TYPE_TO_DOC_MAPPING` dictionary, and compiles a list of errors. The function raises a `ValueError` with a list of error messages if any errors are found.

**Key Implementation Details**

* Iterates through documentation pages under `PATH_TO_DOC/model_doc`
* Uses `transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING` to get existing model types
* Compares each doc page's name with the model types, providing close matches for non-matching names
* Raises an error if any doc pages do not match an existing model type

**Note**

The function relies on external dependencies:

* `transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING`
* `get_close_matches` (not shown in this code snippet)
* `models/auto/configuration_auto`


key: chroma:document
string_value:
CODE:

def check_deprecated_constant_is_up_to_date():
    """
    Check if the constant `DEPRECATED_MODELS` in `models/auto/configuration_auto.py` is up to date.
    """
    deprecated_folder = os.path.join(PATH_TO_TRANSFORMERS, "models", "deprecated")
    deprecated_models = [m for m in os.listdir(deprecated_folder) if not m.startswith("_")]

    constant_to_check = transformers.models.auto.configuration_auto.DEPRECATED_MODELS
    message = []
    missing_models = sorted(set(deprecated_models) - set(constant_to_check))
    if len(missing_models) != 0:
        missing_models = ", ".join(missing_models)
        message.append(
            "The following models are in the deprecated folder, make sure to add them to `DEPRECATED_MODELS` in "
            f"`models/auto/configuration_auto.py`: {missing_models}."
        )

    extra_models = sorted(set(constant_to_check) - set(deprecated_models))
    if len(extra_models) != 0:
        extra_models = ", ".join(extra_models)
        message.append(
            "The following models are in the `DEPRECATED_MODELS` constant but not in the deprecated folder. Either "
            f"remove them from the constant or move to the deprecated folder: {extra_models}."
        )

    if len(message) > 0:


SUMMARY:
**Summary**

The `check_deprecated_constant_is_up_to_date` function checks the consistency of the `DEPRECATED_MODELS` constant in `models/auto/configuration_auto.py`. It ensures that:

* All models in the deprecated folder are included in the constant.
* None of the models listed in the constant are not present in the deprecated folder.

**Key Implementation Details**

* The function uses a list comprehension to extract model names from the deprecated folder.
* It compares the models in the `DEPRECATED_MODELS` constant with those in the deprecated folder using set differences.
* If any inconsistencies are found, it raises an exception with a descriptive message.

**Notable Dependencies and Requirements**

* `transformers`: A dependency for accessing the `models.auto.configuration_auto` module.
* The function assumes that the constants and folders referenced are located within this module.


key: chroma:document
string_value:
CODE:

def check_repo_quality():
    """Check all models are tested and documented."""
    print("Repository-wide checks:")
    print("    - checking all models are included.")
    check_model_list()
    print("    - checking all models are public.")
    check_models_are_in_init()
    print("    - checking all models have tests.")
    check_all_decorator_order()
    check_all_models_are_tested()
    print("    - checking all objects have documentation.")
    check_all_objects_are_documented()
    print("    - checking all models are in at least one auto class.")
    check_all_models_are_auto_configured()
    print("    - checking all names in auto name mappings are defined.")
    check_all_auto_object_names_being_defined()
    print("    - checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")
    check_all_auto_mapping_names_in_config_mapping_names()
    print("    - checking all auto mappings could be imported.")
    check_all_auto_mappings_importable()
    print("    - checking all objects are equally (across frameworks) in the main __init__.")
    check_objects_being_equally_in_main_init()
    print("    - checking the DEPRECATED_MODELS constant is up to date.")


SUMMARY:
**Summary:**

The `check_repo_quality` function runs a sequence of checks on a repository, evaluating various aspects of model quality and configuration. Specifically, it:

*   Verifies models' inclusion
*   Inspects public visibility of models
*   Confirms test existence for models
*   Checks for documentation presence in objects
*   Validates auto-class configurations
*   Examines auto-mapping definitions and importability
*   Ensures object distribution in `__init__`
*   Verifies the `DEPRECATED_MODELS` constant's accuracy

**Key Functions/Methods:**

- `check_model_list()`: Verifies models' existence
- `check_models_are_in_init()`: Inspects public visibility of models
- `check_all_decorator_order()`: Confirms test existence for models
- `check_all_models_are_tested()`: Confirms documentation presence in objects
- `check_all_objects_are_documented()`: Validates auto-mapping definitions and importability

**Notable Dependencies/Requirements:**

- The function relies on the `CONFIG_MAPPING_NAMES` constant, indicating an external dependency.
- It also expects the existence of a `DEPRECATED_MODELS` constant, implying external configuration or setup.

Please note that further context is necessary to provide a more accurate analysis.


key: chroma:document
string_value:
CODE:

def clean_doctest_list(doctest_file: str, overwrite: bool = False):
    """
    Cleans the doctest in a given file.

    Args:
        doctest_file (`str`):
            The path to the doctest file to check or clean.
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to fix problems. If `False`, will error when the file is not clean.
    """
    non_existent_paths = []
    all_paths = []
    with open(doctest_file, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip().split(" ")[0]
            path = os.path.join(REPO_PATH, line)
            if not (os.path.isfile(path) or os.path.isdir(path)):
                non_existent_paths.append(line)
            all_paths.append(line)

    if len(non_existent_paths) > 0:
        non_existent_paths = "\n".join([f"- {f}" for f in non_existent_paths])
        raise ValueError(f"`{doctest_file}` contains non-existent paths:\n{non_existent_paths}")

    sorted_paths = sorted(all_paths)
    if all_paths != sorted_paths:
        if not overwrite:
            raise ValueError(
                f"Files in `{doctest_file}` are not in alphabetical order, run `make fix-copies` to fix "
                "this automatically."
            )
        with open(doctest_file, "w", encoding="utf-8") as f:


SUMMARY:
**Code Summary**

### Function Purpose

`clean_doctest_list` cleans and organizes a doctest file by removing non-existent paths and sorting the remaining paths in alphabetical order.

### Key Implementation Details

* The function reads a doctest file line-by-line, stripping leading whitespace and splitting on spaces.
* It checks if each path exists and creates a list of non-existent paths.
* If any non-existent paths are found, they are reported as an error.
* The function then sorts the remaining paths alphabetically and writes them back to the original file. The sorting can be skipped with the `overwrite` parameter.

### Notable Dependencies

* `os.path`: used for path manipulation (e.g., `join`, `resolve`)
* `REPO_PATH`: assumed to be a predefined constant containing the repository root path

### Method Signature and Interfaces

| Method/Interface | Description |
| --- | --- |
| `clean_doctest_list(doctest_file: str, overwrite: bool = False)` | Cleans and organizes a doctest file. |

### External Dependencies

None mentioned in this snippet, but `os` module is used implicitly.


key: chroma:document
string_value:
CODE:

def _is_definition_header_ending_line(line: str) -> bool:
    # Helper function. Returns `True` if `line` is the end parenthesis of a class/function definition


SUMMARY:
**Summary:**

This function, `_is_definition_header_ending_line`, determines whether a given line of code indicates the end of a class or function definition.

```markdown
### Main Purpose and Functionality

Determines if a line ends an opening parenthesis of a class or function definition.

### Key Implementation Details

- Uses regular expression (`re.search`) to search for the pattern `^\s*\)(\s*->.*:|:)\s*$` at the start of the line.
- Returns `True` when the pattern is found, indicating an ending definition line.
- The primary purpose is to identify lines that signal the completion of a class or function definition.

### Notable Dependencies and Requirements

None listed, but uses Python's built-in `re` module for regular expression matching.
```

**Additional Information:**

This function does not depend on any external libraries beyond the `re` module.


key: chroma:document
string_value:
CODE:

def _should_continue(line: str, indent: str) -> bool:
    # Helper function. Returns `True` if `line` is empty, starts with the `indent` or is the end parenthesis of a
    # class/function definition


SUMMARY:
**Summary:**

- `should_continue(line: str, indent: str) -> bool`: Returns True if the given line is empty, starts with the specified indent, or marks the end of a class or function definition.

**Implementation Details:**

- Checks for three conditions:
  - Starts with the specified indent (`line.startswith(indent)`)
  - Is an empty line after stripping whitespace (`len(line.strip()) == 0`)
  - Marks the end of a definition header using `_is_definition_header_ending_line(line)`


key: chroma:document
string_value:
CODE:

def _sanity_check_splits(splits_1, splits_2, is_class, filename):
    """Check the two (inner) block structures of the corresponding code block given by `split_code_into_blocks` match.

    For the case of `class`, they must be of one of the following 3 cases:

        - a single block without name:

            class foo:
                a = 1

        - a consecutive sequence of (1 or more) blocks with name

            class foo:

                def f(x):
                    return x

        - a block without name, followed by a consecutive sequence of (1 or more) blocks with name

            class foo:
                a = 1

                def f(x):
                    return x

                def g(x):
                    return None

    The 2 code snippets that give `splits_1` and `splits_2` have to be in the same case to pass this check, but the
    number of blocks with name in the consecutive sequence is not taken into account.

    For the case of `function or method`, we don't require it to be in one of the above 3 cases. However, the structure
    of`splits_1` and `splits_2` have to match exactly. In particular, the number of blocks with name in a consecutive
    sequence is taken into account.
    """
    block_names_1 = []
    block_names_2 = []

    for block in splits_1[1:]:
        if block[0].startswith("_block_without_name_"):
            block_names_1.append("block_without_name")
        elif not block[0].startswith("_empty_block_") and (
            not is_class or len(block_names_1) == 0 or block_names_1[-1].startswith("block_without_name")
        ):
            block_names_1.append("block_with_name")

    for block in splits_2[1:]:
        if block[0].startswith("_block_without_name_"):
            block_names_2.append("block_without_name")
        elif not block[0].startswith("_empty_block_") and (
            not is_class or len(block_names_2) == 0 or block_names_2[-1].startswith("block_without_name")
        ):
            block_names_2.append("block_with_name")

    if is_class:
        if block_names_1 not in [
            ["block_without_name"],
            ["block_with_name"],
            ["block_without_name", "block_with_name"],
        ]:
            raise ValueError(
                f"""Class defined in {filename} doesn't have the expected stucture.
                See the docstring of `_sanity_check_splits` in the file `utils/check_copies.py`""",
            )

    if block_names_1 != block_names_2:


SUMMARY:
**Function Summary: `_sanity_check_splits`**

### Purpose and Functionality

Verifies the structure of paired code blocks in `splits_1` and `splits_2`. For classes, ensures one of three predefined block structures is matched. For functions/methods, verifies exact matching of block arrangements.

### Key Implementation Details

* Class block structure checking:
	+ Supports single block without name and consecutive sequence of blocks with name
	+ Number of blocks with name in consecutive sequence not considered
* Function/method block structure checking:
	+ Exact matching of block arrangement required

### Notable Dependencies and Requirements

* `is_class` flag indicating class definition or not
* `filename` parameter for logging purposes
* Internal constants and interface (`_block_without_name_`, `_empty_block_`)


key: chroma:document
string_value:
CODE:

def find_block_end(lines: List[str], start_index: int, indent: int) -> int:
    """
    Find the end of the class/func block starting at `start_index` in a source code (defined by `lines`).

    Args:
        lines (`List[str]`):
            The source code, represented by a list of lines.
        start_index (`int`):
            The starting index of the target class/func block.
        indent (`int`):
            The indent of the class/func body.

    Returns:
        `int`: The index of the block's ending line plus by 1 (i.e. exclusive).
    """
    indent = " " * indent
    # enter the block body
    line_index = start_index + 1

    while line_index < len(lines) and _should_continue(lines[line_index], indent):
        line_index += 1
    # Clean up empty lines at the end (if any).
    while len(lines[line_index - 1]) <= 1:
        line_index -= 1



SUMMARY:
**Function Summary:**
`find_block_end`
---------------

Find the index of the block's ending line in a source code.

```python
def find_block_end(lines: List[str], start_index: int, indent: int) -> int:
    # ...
```

### Purpose and Functionality:

This function finds the end of a class/func block in a source code, starting from `start_index`.
It iterates through the lines until it encounters an invalid continuation line or reaches the end of the code.

### Key Implementation Details:

- Iterates until a blank or invalid line (`_should_continue`) is encountered.
- Cleans up empty lines at the end.
- Returns the index of the block's ending line (inclusive).

### Dependencies and Requirements:
 No external dependencies.


key: chroma:document
string_value:
CODE:

def split_code_into_blocks(
    lines: List[str], start_index: int, end_index: int, indent: int, backtrace: bool = False
) -> List[Tuple[str, int, int]]:
    """
    Split the class/func block starting at `start_index` in a source code (defined by `lines`) into *inner blocks*.

    The block's header is included as the first element. The contiguous regions (without empty lines) that are not
    inside any inner block are included as blocks. The contiguous regions of empty lines that are not inside any inner
    block are also included as (dummy) blocks.

    Args:
        lines (`List[str]`):
            The source code, represented by a list of lines.
        start_index (`int`):
            The starting index of the target class/func block.
        end_index (`int`):
            The ending index of the target class/func block.
        indent (`int`):
            The indent of the class/func body.
        backtrace (`bool`, *optional*, defaults to `False`):
            Whether or not to include the lines before the inner class/func block's header (e.g. comments, decorators,
            etc.) until an empty line is encountered.

    Returns:
        `List[Tuple[str, int, int]]`: A list of elements with the form `(block_name, start_index, end_index)`.
    """
    splits = []
    # `indent - 4` is the indent level of the target class/func header
    try:
        target_block_name = re.search(
            rf"^{' ' * (indent - 4)}((class|def)\s+\S+)(\(|\:)", lines[start_index]
        ).groups()[0]
    except Exception:
        start_context = min(start_index - 10, 0)
        end_context = min(end_index + 10, len(lines))
        raise ValueError(
            f"Tried to split a class or function. It did not work. Error comes from line {start_index}: \n```\n"
            + "".join(lines[start_context:end_context])
            + "```\n"
        )

    # from now on, the `block` means inner blocks unless explicitly specified
    indent_str = " " * indent
    block_without_name_idx = 0
    empty_block_idx = 0

    # Find the lines for the definition header
    index = start_index
    if "(" in lines[start_index] and "):" not in lines[start_index] in lines[start_index]:
        while index < end_index:
            if _is_definition_header_ending_line(lines[index]):
                break
            index += 1

    # the first line outside the definition header
    index += 1
    splits.append((target_block_name, start_index, index))

    block_start_index, prev_block_end_index = index, index
    while index < end_index:
        # if found, it will be an inner block
        block_found = re.search(rf"^{indent_str}((class|def)\s+\S+)(\(|\:)", lines[index])
        if block_found:
            name = block_found.groups()[0]

            block_end_index = find_block_end(lines, index, indent + 4)

            # backtrace to include the lines before the found block's definition header (e.g. comments, decorators,
            # etc.) until an empty line is encountered.
            block_start_index = index
            if index > prev_block_end_index and backtrace:
                idx = index - 1
                for idx in range(index - 1, prev_block_end_index - 2, -1):
                    if not (len(lines[idx].strip()) > 0 and lines[idx].startswith(indent_str)):
                        break
                idx += 1
                if idx < index:
                    block_start_index = idx

            # between the current found block and the previous found block
            if block_start_index > prev_block_end_index:
                # give it a dummy name
                if len("".join(lines[prev_block_end_index:block_start_index]).strip()) == 0:
                    prev_block_name = f"_empty_block_{empty_block_idx}"
                    empty_block_idx += 1
                else:
                    prev_block_name = f"_block_without_name_{block_without_name_idx}"
                    block_without_name_idx += 1
                # Add it as a block
                splits.append((prev_block_name, prev_block_end_index, block_start_index))

            # Add the current found block
            splits.append((name, block_start_index, block_end_index))
            prev_block_end_index = block_end_index
            index = block_end_index - 1

        index += 1

    if index > prev_block_end_index:
        if len("".join(lines[prev_block_end_index:index]).strip()) == 0:
            prev_block_name = f"_empty_block_{empty_block_idx}"
        else:
            prev_block_name = f"_block_without_name_{block_without_name_idx}"
        splits.append((prev_block_name, prev_block_end_index, index))



SUMMARY:
**Function Summary: `split_code_into_blocks`**

### Purpose and Functionality
Splits the code block in the specified range into inner blocks, including contiguous regions without empty lines.

### Key Implementation Details

* Takes source code `_lines`, start index `_start_index`, end index `_end_index`, and `indent_level`.
* Uses a regular expression to find the definition header.
* Iterates through the code to find inner blocks using a regular expression.
* Handles backtrace by including lines before the inner block's definition header.

### Method Signatures and Interfaces

* Takes a list of strings (`lines`) as input.
* Returns a list of tuples containing string, start index, and end index (`List[Tuple[str, int, int]]`).

### Notable Dependencies or Requirements

* Uses the `re` module for regular expressions.

### Output Format
Returns a list of tuples containing block names, start indices, and end indices.

### Summary

Splits code into inner blocks using a regular expression-based approach. It handles backtrace and returns a list of tuples containing block information.


key: chroma:document
string_value:
CODE:

def find_code_in_transformers(
    object_name: str, base_path: str = None, return_indices: bool = False
) -> Union[str, Tuple[List[str], int, int]]:
    """
    Find and return the source code of an object.

    Args:
        object_name (`str`):
            The name of the object we want the source code of.
        base_path (`str`, *optional*):
            The path to the base folder where files are checked. If not set, it will be set to `TRANSFORMERS_PATH`.
        return_indices(`bool`, *optional*, defaults to `False`):
            If `False`, will only return the code (as a string), otherwise it will also return the whole lines of the
            file where the object specified by `object_name` is defined, together the start/end indices of the block in
            the file that defines the object.

    Returns:
        `Union[str, Tuple[List[str], int, int]]`: If `return_indices=False`, only the source code of the object will be
        returned. Otherwise, it also returns the whole lines of the file where the object specified by `object_name` is
        defined, together the start/end indices of the block in the file that defines the object.
    """
    parts = object_name.split(".")
    i = 0

    # We can't set this as the default value in the argument, otherwise `CopyCheckTester` will fail, as it uses a
    # patched temp directory.
    if base_path is None:
        base_path = TRANSFORMERS_PATH

    # Detail: the `Copied from` statement is originally designed to work with the last part of `TRANSFORMERS_PATH`,
    # (which is `transformers`). The same should be applied for `MODEL_TEST_PATH`. However, its last part is `models`
    # (to only check and search in it) which is a bit confusing. So we keep the copied statement staring with
    # `tests.models.` and change it to `tests` here.
    if base_path == MODEL_TEST_PATH:
        base_path = "tests"

    # First let's find the module where our object lives.
    module = parts[i]
    while i < len(parts) and not os.path.isfile(os.path.join(base_path, f"{module}.py")):
        i += 1
        if i < len(parts):
            module = os.path.join(module, parts[i])
    if i >= len(parts):
        raise ValueError(
            f"`object_name` should begin with the name of a module of transformers but got {object_name}."
        )

    with open(os.path.join(base_path, f"{module}.py"), "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()

    # Now let's find the class / func in the code!
    indent = ""
    line_index = 0
    for name in parts[i + 1 :]:
        while (
            line_index < len(lines) and re.search(rf"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None
        ):
            line_index += 1
        # find the target specified in the current level in `parts` -> increase `indent` so we can search the next
        indent += "    "
        # the index of the first line in the (currently found) block *body*
        line_index += 1

    if line_index >= len(lines):
        raise ValueError(f" {object_name} does not match any function or class in {module}.")

    # `indent` is already one level deeper than the (found) class/func block's definition header

    # We found the beginning of the class / func, now let's find the end (when the indent diminishes).
    # `start_index` is the index of the class/func block's definition header
    start_index = line_index - 1
    end_index = find_block_end(lines, start_index, len(indent))

    code = "".join(lines[start_index:end_index])


SUMMARY:
**Summary: `find_code_in_transformers` Function**

The `find_code_in_transformers` function finds and returns the source code of an object in the transformers library.

### Purpose and Functionality

* Takes an object name, base path (optional), and return indices as input arguments.
* Returns either the object's source code or a tuple containing the source code, file lines, start index, and end index.

### Key Implementation Details

* Finds the module where the object lives by splitting the object name and checking existing files in the base path.
* Parses the Python code to find the class or function specified by the object name, returning the source code if `return_indices` is False.

### Notable Dependencies and Requirements

* Uses external dependencies:
	+ `os` for file operations
	+ `re` for regular expression matching
	+ `transformers` library (not explicitly imported)
* Inherited or implemented interfaces:
	+ None, but uses Python's built-in functions such as `open`, `readlines`, and `find_block_end`.

### Method Signatures

The function takes the following input arguments:

* `object_name`: str
* `base_path`: str (optional)
* `return_indices`: bool (optional)

The function returns a Union type containing either str or a tuple with the following elements:
	+ str: source code of the object, if `return_indices` is False.
	+ Tuple[List[str], int, int]: file lines, start index, and end index, if `return_indices` is True.


key: chroma:document
string_value:
CODE:

def replace_code(code: str, replace_pattern: str) -> str:
    """Replace `code` by a pattern of the form `with X1->X2,Y1->Y2,Z1->Z2`.

    Args:
        code (`str`): The code to be modified.
        replace_pattern (`str`): The pattern used to modify `code`.

    Returns:
        `str`: The modified code.
    """
    if len(replace_pattern) > 0:
        patterns = replace_pattern.replace("with", "").split(",")
        patterns = [_re_replace_pattern.search(p) for p in patterns]
        for pattern in patterns:
            if pattern is None:
                continue
            obj1, obj2, option = pattern.groups()
            code = re.sub(obj1, obj2, code)
            if option.strip() == "all-casing":
                code = re.sub(obj1.lower(), obj2.lower(), code)
                code = re.sub(obj1.upper(), obj2.upper(), code)



SUMMARY:
**Summary**

`replace_code` function:

* Replaces occurrences of `obj1` with `obj2` in `code`, optionally applying casing conversions (`all-casing` option).
* Takes two inputs: `code` (string) and `replace_pattern` (string, formatted as "with X1->X2,Y1->Y2,Z1->Z2").
* Utilizes regular expressions to perform replacements, leveraging `_re_replace_pattern.search()` and `re.sub()` functions.

**Key Implementation Details**

* Uses regular expression substitutions (`re.sub()`) to replace patterns in the input code.
* Applies casing conversions for the "all-casing" option using additional `re.sub()` calls.
* Utilizes `_re_replace_pattern.search()` to extract pattern groups (e.g., `obj1`, `obj2`, and `option`).


key: chroma:document
string_value:
CODE:

def find_code_and_splits(object_name: str, base_path: str, buffer: dict = None):
    """Find the code of an object (specified by `object_name`) and split it into blocks.

    Args:
        object_name (`str`):
            The name of the object, e.g. `transformers.models.bert.modeling_bert.BertAttention` or
            `tests.models.llama.test_modeling_llama.LlamaModelTest.test_config`.
        base_path (`str`):
            The path to the base directory within which the search will be performed. It could be either
            `TRANSFORMERS_PATH` or `MODEL_TEST_PATH`.
        buffer (`dict`, *optional*):
            The buffer used to store the previous results in order to speed up the process.

    Returns:
        lines (`List[str]`):
            The lines of the whole file where the object is defined.
        code (`str`):
            The object's code.
        code_splits (`List[Tuple[str, int, int]]`):
            `code` splitted into blocks. See `split_code_into_blocks`.
    """
    if buffer is None:
        buffer = {}

    if (object_name, base_path) in buffer:
        lines, code, code_splits = buffer[(object_name, base_path)]
    else:
        code, (lines, target_start_index, target_end_index) = find_code_in_transformers(
            object_name, base_path=base_path, return_indices=True
        )
        indent = get_indent(code)

        # Split the code into blocks
        # `indent` is the indent of the class/func definition header, but `code_splits` expects the indent level of the
        # block body.
        code_splits = split_code_into_blocks(
            lines, target_start_index, target_end_index, len(indent) + 4, backtrace=True
        )
        buffer[(object_name, base_path)] = lines, code, code_splits



SUMMARY:
**Code Summary: `find_code_and_splits` Function**

**Purpose and Functionality:**
The `find_code_and_splits` function searches for the code of an object within a specified directory (`base_path`) and splits it into blocks. It retrieves previous results from a buffer to speed up the process.

**Key Implementation Details:**

*   Uses `find_code_in_transformers` to search for the code in the transformers library.
*   Utilizes `split_code_into_blocks` to split the retrieved code into blocks based on indentation levels.
*   Expects the `buffer` parameter to store previous results and retrieve them if available.

**Notable Dependencies and Requirements:**

*   `find_code_in_transformers`: Function from transformers library or custom implementation.
*   `split_code_into_blocks`: Custom or third-party function for block splitting.
*   `get_indent`: Custom or external function to calculate indentation levels.


key: chroma:document
string_value:
CODE:

def get_indent(code: str) -> str:
    """
    Find the indent in the first non empty line in a code sample.

    Args:
        code (`str`): The code to inspect.

    Returns:
        `str`: The indent looked at (as string).
    """
    lines = code.split("\n")
    idx = 0
    while idx < len(lines) and len(lines[idx]) == 0:
        idx += 1
    if idx < len(lines):
        return re.search(r"^(\s*)\S", lines[idx]).groups()[0]


SUMMARY:
**Function Summary**

GetIndent Function: Finds the indent in the first non-empty line of code.

**Primary Purpose:** Extracts the leading whitespace from the first non-empty line of code.

**Key Implementation Details:**

* Splits code into individual lines
* Iterates through lines until a non-empty one is found
* Uses regex to match and extract the leading whitespace

**Notable Dependencies:**

* `re` module for regular expression matching ( referencing `search()` method)
* No external dependencies listed, but assumes Python environment with `str` and `list` data types.

**Method Signature:** `get_indent(code: str) -> str`

This function is designed to be used in a context where code snippets need to be analyzed or processed before execution. It provides a concise way to extract the leading whitespace from the first non-empty line of code, making it easier to analyze or visualize indentation schemes.


key: chroma:document
string_value:
CODE:

def run_ruff(code, check=False):
    if check:
        command = ["ruff", "check", "-", "--fix", "--exit-zero"]
    else:
        command = ["ruff", "format", "-", "--config", "pyproject.toml", "--silent"]
    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
    stdout, _ = process.communicate(input=code.encode())


SUMMARY:
**Summary:**

`run_ruff` is a function that runs the Ruff code formatter with either `check` or `format` functionality. It takes in code and configuration options, utilizes subprocess to execute the Ruff command, and returns the formatted output.

**Key Implementation Details:**

- Utilizes `subprocess.Popen` for executing the Ruff command
- Supports both `check` (linting) and `format` (code reformattting)
- Uses `pyproject.toml` as default configuration file

**Dependences/Requirements:**

- Ruff code formatter (`ruff`)
- `pyproject.toml`

Note: The function returns the formatted output from the standard output of the Ruff command.


key: chroma:document
string_value:
CODE:

def stylify(code: str) -> str:
    """
    Applies the ruff part of our `make style` command to some code. This formats the code using `ruff format`.
    As `ruff` does not provide a python api this cannot be done on the fly.

    Args:
        code (`str`): The code to format.

    Returns:
        `str`: The formatted code.
    """
    has_indent = len(get_indent(code)) > 0
    if has_indent:
        code = f"class Bla:\n{code}"
    formatted_code = run_ruff(code)


SUMMARY:
Here is a concise summary of the `stylify` function:

**Summary:**
Converts input Python code to be compatible with RUFF's formatting style.

**Key Details:**

* Formats input code using the `ruff format` command
* Appends an empty class wrapper to preserve indentation if present
* Returns the formatted code without the wrapped class prefix

**Notable Dependencies:**

* `run_ruff`: External function from external library (not shown)
* `get_indent`: External function from external library (not shown)


key: chroma:document
string_value:
CODE:

def check_codes_match(observed_code: str, theoretical_code: str) -> Optional[int]:
    """
    Checks if two version of a code match with the exception of the class/function name.

    Args:
        observed_code (`str`): The code found.
        theoretical_code (`str`): The code to match.

    Returns:
        `Optional[int]`: The index of the first line where there is a difference (if any) and `None` if the codes
        match.
    """
    observed_code_header = observed_code.split("\n")[0]
    theoretical_code_header = theoretical_code.split("\n")[0]

    # Catch the function/class name: it is expected that those do not match.
    _re_class_match = re.compile(r"class\s+([^\(:]+)(?:\(|:)")
    _re_func_match = re.compile(r"def\s+([^\(]+)\(")
    for re_pattern in [_re_class_match, _re_func_match]:
        if re_pattern.match(observed_code_header) is not None:
            try:
                observed_obj_name = re_pattern.search(observed_code_header).groups()[0]
            except Exception:
                raise ValueError(
                    "Tried to split a class or function. It did not work. Error comes from: \n```\n"
                    + observed_code_header
                    + "\n```\n"
                )

            try:
                theoretical_name = re_pattern.search(theoretical_code_header).groups()[0]
            except Exception:
                raise ValueError(
                    "Tried to split a class or function. It did not work. Error comes from: \n```\n"
                    + theoretical_code_header
                    + "\n```\n"
                )
            theoretical_code_header = theoretical_code_header.replace(theoretical_name, observed_obj_name)

    # Find the first diff. Line 0 is special since we need to compare with the function/class names ignored.
    diff_index = 0
    if theoretical_code_header != observed_code_header:
        return 0

    diff_index = 1
    for observed_line, theoretical_line in zip(observed_code.split("\n")[1:], theoretical_code.split("\n")[1:]):
        if observed_line != theoretical_line:
            return diff_index


SUMMARY:
**Function Summary**

`check_codes_match`

Checks if two versions of a code match with the exception of class/function name.

**Purpose and Functionality**

Compares two code strings, ignoring class/function names. Returns the index of the first line where there is a difference (if any) or `None` if they match.

**Key Implementation Details**

* Uses regular expressions to extract class/function names from header lines.
* Compares each line in the code separately, ignoring the extracted names.

**Notable Dependencies or Requirements**

* Python's `re` module for regex pattern matching
* Python's `Optional` type hinting

**Return Value**

`Optional[int]`: The index of the first difference or `None` (no difference)

**Method Signatures and Interfaces**

* `_re_class_match` and `_re_func_match` regular expression patterns for class/function name extraction.

Note: This summary focuses on the most important information about the function, omitting minor details.


key: chroma:document
string_value:
CODE:

def is_copy_consistent(filename: str, overwrite: bool = False, buffer: dict = None) -> Optional[List[Tuple[str, int]]]:
    """
    Check if the code commented as a copy in a file matches the original.

    Args:
        filename (`str`):
            The name of the file to check.
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
        buffer (`dict`, *optional*):
            The buffer used to store the previous results in order to speed up the process.

    Returns:
        `Optional[List[Tuple[str, int]]]`: If `overwrite=False`, returns the list of differences as tuples `(str, int)`
        with the name of the object having a diff and the line number where theere is the first diff.
    """
    base_path = TRANSFORMERS_PATH if not filename.startswith("tests") else MODEL_TEST_PATH

    with open(filename, "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()
    diffs = []
    line_index = 0
    # Not a for loop cause `lines` is going to change (if `overwrite=True`).
    search_re = _re_copy_warning_for_test_file if filename.startswith("tests") else _re_copy_warning
    while line_index < len(lines):
        search = search_re.search(lines[line_index])
        if search is None:
            line_index += 1
            continue

        # There is some copied code here, let's retrieve the original.
        indent, object_name, replace_pattern = search.groups()

        # Find the file lines, the object's code, and its blocks
        try:
            target_lines, theoretical_code, theoretical_code_splits = find_code_and_splits(
                object_name, base_path, buffer=buffer
            )
        except Exception as exc:
            exc.args = (f"Error while trying to find source code for {filename}.\n\n" + str(exc),)
            raise

        # code replaced by the patterns
        theoretical_code_blocks = OrderedDict()
        for name, start, end in theoretical_code_splits:
            name = replace_code(name, replace_pattern)
            code = "".join(target_lines[start:end])
            code = replace_code(code, replace_pattern)
            theoretical_code_blocks[name] = code

        theoretical_indent = get_indent(theoretical_code)

        # `start_index` is the index of the first line (the definition header) after `# Copied from`.
        # (`indent != theoretical_indent` doesn't seem to occur so far, not sure what this case is for.)
        start_index = line_index + 1 if indent == theoretical_indent else line_index
        # enter the block body
        line_index = start_index + 1

        subcode = "\n".join(theoretical_code.split("\n")[1:])
        indent = get_indent(subcode)
        # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.
        # We can't call `find_block_end` directly as there is sth. special `# End copy"` here.
        should_continue = True
        while line_index < len(lines) and should_continue:
            line_index += 1
            if line_index >= len(lines):
                break
            line = lines[line_index]
            # There is a special pattern `# End copy` to stop early. It's not documented cause it shouldn't really be
            # used.
            should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None
        # `line_index` is outside the block
        # Clean up empty lines at the end (if any).
        while len(lines[line_index - 1]) <= 1:
            line_index -= 1

        # Split the observed code into blocks
        observed_code_splits = split_code_into_blocks(lines, start_index, line_index, len(indent), backtrace=True)

        is_class = lines[start_index].startswith(f"{' ' * (len(indent) - 4)}class ")
        # sanity check
        _sanity_check_splits(theoretical_code_splits, observed_code_splits, is_class=is_class, filename=filename)

        # observed code in a structured way (a dict mapping block names to blocks' code)
        observed_code_blocks = OrderedDict()
        for name, start, end in observed_code_splits:
            code = "".join(lines[start:end])
            observed_code_blocks[name] = code

        # Below, we change some names in `theoretical_code_blocks` and `observed_code_blocks`. These mappings map the
        # original names to the modified names: this is used to restore the original order of the code blocks.
        name_mappings_1 = {k: k for k in theoretical_code_blocks.keys()}
        name_mappings_2 = {k: k for k in observed_code_blocks.keys()}

        # Update code blocks' name and content:
        #   If `"# Ignore copy"` is found in a block of the observed code:
        #     1. if it's a block only in the observed code --> add it to the theoretical code.
        #     2. if it's also in the theoretical code () --> put its content (body) to the corresponding block under the
        #        same name in the theoretical code.
        #   In both cases, we change the name to have a prefix `_ignored_` so we know if we can discard them during the
        #   comparison.
        ignored_existing_block_index = 0
        ignored_new_block_index = 0
        for name in list(observed_code_blocks.keys()):
            code = observed_code_blocks[name]
            if "# Ignore copy" in code:
                if name in theoretical_code_blocks:
                    # in the target --> just copy the content
                    del theoretical_code_blocks[name]
                    theoretical_code_blocks[f"_ignored_existing_block_{ignored_existing_block_index}"] = code
                    name_mappings_1[name] = f"_ignored_existing_block_{ignored_existing_block_index}"

                    del observed_code_blocks[name]
                    observed_code_blocks[f"_ignored_existing_block_{ignored_existing_block_index}"] = code
                    name_mappings_2[name] = f"_ignored_existing_block_{ignored_existing_block_index}"
                    ignored_existing_block_index += 1
                else:
                    # not in the target --> add it
                    theoretical_code_blocks[f"_ignored_new_block_{ignored_new_block_index}"] = code
                    name_mappings_1[f"_ignored_new_block_{ignored_new_block_index}"] = (
                        f"_ignored_new_block_{ignored_new_block_index}"
                    )

                    del observed_code_blocks[name]
                    observed_code_blocks[f"_ignored_new_block_{ignored_new_block_index}"] = code
                    name_mappings_2[name] = f"_ignored_new_block_{ignored_new_block_index}"
                    ignored_new_block_index += 1

        # Respect the original block order:
        #   1. in `theoretical_code_blocks`: the new blocks will follow the existing ones
        #   2. in `observed_code_blocks`: the original order are kept with names modified potentially. This is necessary
        #      to compute the correct `diff_index` if `overwrite=True` and there is a diff.
        theoretical_code_blocks = {
            name_mappings_1[orig_name]: theoretical_code_blocks[name_mappings_1[orig_name]]
            for orig_name in name_mappings_1
        }
        observed_code_blocks = {
            name_mappings_2[orig_name]: observed_code_blocks[name_mappings_2[orig_name]]
            for orig_name in name_mappings_2
        }

        # Ignore the blocks specified to be ignored. This is the version used to check if there is a mismatch
        theoretical_code_blocks_clean = {
            k: v
            for k, v in theoretical_code_blocks.items()
            if not (k.startswith(("_ignored_existing_block_", "_ignored_new_block_")))
        }
        theoretical_code = "".join(list(theoretical_code_blocks_clean.values()))

        # stylify `theoretical_code` before compare (this is needed only when `replace_pattern` is not empty)
        if replace_pattern:
            theoretical_code = stylify(theoretical_code)
        # Remove `\n\n` in `theoretical_code` before compare (so no empty line)
        while "\n\n" in theoretical_code:
            theoretical_code = theoretical_code.replace("\n\n", "\n")

        # Compute `observed_code` where we don't include any empty line + keep track the line index between the
        # original/processed `observed_code` so we can have the correct `diff_index`.
        idx_to_orig_idx_mapping_for_observed_code_lines = {}
        idx = -1
        orig_idx = -1
        observed_code = ""
        for name, code in observed_code_blocks.items():
            if code.endswith("\n"):
                code = code[:-1]
            for code_line in code.split("\n"):
                orig_idx += 1
                if code_line.strip() and not name.startswith(("_ignored_existing_block_", "_ignored_new_block_")):
                    idx += 1
                    observed_code += code_line + "\n"
                    idx_to_orig_idx_mapping_for_observed_code_lines[idx] = orig_idx

        # Test for a diff and act accordingly.
        diff_index = check_codes_match(observed_code, theoretical_code)
        if diff_index is not None:
            # switch to the index in the original `observed_code` (i.e. before removing empty lines)
            diff_index = idx_to_orig_idx_mapping_for_observed_code_lines[diff_index]
            diffs.append([object_name, diff_index + start_index + 1])
            if overwrite:
                # `theoretical_code_to_write` is a single string but may have several lines.
                theoretical_code_to_write = stylify("".join(list(theoretical_code_blocks.values())))
                lines = lines[:start_index] + [theoretical_code_to_write] + lines[line_index:]
                # Here we treat it as a single entry in `lines`.
                line_index = start_index + 1

    if overwrite and len(diffs) > 0:
        # Warn the user a file has been modified.
        print(f"Detected changes, rewriting {filename}.")
        with open(filename, "w", encoding="utf-8", newline="\n") as f:
            f.writelines(lines)


SUMMARY:
This is a function that compares two versions of a code file. It checks if the code has been modified and reports changes. The function takes into account various factors such as:

* Overwriting existing files
* Handling empty lines and whitespace differences
* Respecting the original block order when ignoring certain blocks
* Stylifying codes for comparison

Here's a brief overview of what the function does:

1. It reads two versions of the code file, comparing them line by line.
2. If there are changes between the two versions, it reports these changes and tracks their corresponding indices in both versions.
3. Depending on whether to overwrite existing files or not, it either makes the necessary modifications or simply returns a list of reported differences.
4. It also provides an option for modifying the code file in-place if there are any detected changes.

The function aims to be efficient and accurate when comparing two versions of code and can handle a range of scenarios, from minor changes to major rewrites.


key: chroma:document
string_value:
CODE:

def check_copies(overwrite: bool = False, file: str = None):
    """
    Check every file is copy-consistent with the original. Also check the model list in the main README and other
    READMEs are consistent.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
        file (`bool`, *optional*):
            The path to a specific file to check and/or fix.
    """
    buffer = {}

    if file is None:
        all_files = glob.glob(os.path.join(TRANSFORMERS_PATH, "**/*.py"), recursive=True)
        all_test_files = glob.glob(os.path.join(MODEL_TEST_PATH, "**/*.py"), recursive=True)
        all_files = list(all_files) + list(all_test_files)
    else:
        all_files = [file]

    diffs = []
    for filename in all_files:
        new_diffs = is_copy_consistent(filename, overwrite, buffer)
        diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]
    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
            "Found the following copy inconsistencies:\n"
            + diff
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."


SUMMARY:
**Summary**

The `check_copies` function ensures that all files in specified directories (`TRANSFORMERS_PATH` and `MODEL_TEST_PATH`) are copy-consistent with their original versions. It checks for discrepancies by calling the `is_copy_consistent` function and reports any inconsistencies via an exception.

**Key Takeaways**

* Function: `check_copies`
* Purpose: Validate file copies and report inconsistencies
* Parameters:
	+ `overwrite`: Boolean flag to overwrite discrepancies (default: False)
	+ `file`: Optional path to a single file for manual inspection (defaults to None)

**Implementation Details**

* Iterates through a list of files in specified directories (`all_files`)
* Uses `is_copy_consistent` function to check copy consistency for each file
* Reports inconsistencies via an exception if overwrite is False and discrepancies are found

**Notable Dependencies/Interfaces**

* `os.path.join`
* `glob`
* ` TRANSFORMERS_PATH`
* `MODEL_TEST_PATH`
* `is_copy_consistent` (function, dependencies not specified)


key: chroma:document
string_value:
CODE:

def check_full_copies(overwrite: bool = False):
    """
    Check the files that are full copies of others (as indicated in `FULL_COPIES`) are copy-consistent.

    Args:
        overwrite (`bool`, *optional*, defaults to `False`):
            Whether or not to overwrite the copies when they don't match.
    """
    diffs = []
    for target, source in FULL_COPIES.items():
        with open(source, "r", encoding="utf-8") as f:
            source_code = f.read()
        with open(target, "r", encoding="utf-8") as f:
            target_code = f.read()
        if source_code != target_code:
            if overwrite:
                with open(target, "w", encoding="utf-8") as f:
                    print(f"Replacing the content of {target} by the one of {source}.")
                    f.write(source_code)
            else:
                diffs.append(f"- {target}: copy does not match {source}.")

    if not overwrite and len(diffs) > 0:
        diff = "\n".join(diffs)
        raise Exception(
            "Found the following copy inconsistencies:\n"
            + diff
            + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."


SUMMARY:
**Function Summary: check_full_copies**

 Checks files indicated as full copies (`FULL_COPIES`) for consistency and raises an exception if the copies differ.

**Functionality**

* Compares source code with target copy
* If the target copy differs, considers overwriting the target (if `overwrite` is True) or logs a difference (if `overwrite` is False)
* Rises an exception if no overwrites is made and differences are found

**Key Implementation Details:**

* Uses dictionary `FULL_COPIES` to map source files to their corresponding target files
* Reads file contents using open() with encoding="utf-8"
* Compares source code (`source_code`) with target code (`target_code`)
* If the copies differ, uses a list to track differences (`diffs`)
* Uses exception handling to raise an error if no overwrites is made and differences are found

**Notable Dependencies/Requirements:**

* Python 3.7+ (encoding support)
* Makefile or Python script (utils/check_copies.py)

**Important Method Signatures and Interfaces:**

* `check_full_copies()`: Function to check files for consistency
	+ `overwrite**: Boolean flag to control overwrite behavior
	+ `FULL_COPIES`: Dictionary mapping source files to target files


key: chroma:document
string_value:
CODE:

def get_model_list(filename: str, start_prompt: str, end_prompt: str) -> str:
    """
    Extracts the model list from a README.

    Args:
        filename (`str`): The name of the README file to check.
        start_prompt (`str`): The string to look for that introduces the model list.
        end_prompt (`str`): The string to look for that ends the model list.

    Returns:
        `str`: The model list.
    """
    with open(os.path.join(REPO_PATH, filename), "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()
    # Find the start of the list.
    start_index = 0
    while not lines[start_index].startswith(start_prompt):
        start_index += 1
    start_index += 1

    result = []
    current_line = ""
    end_index = start_index

    # Keep going until the end of the list.
    while not lines[end_index].startswith(end_prompt):
        if lines[end_index].startswith("1."):
            if len(current_line) > 1:
                result.append(current_line)
            current_line = lines[end_index]
        elif len(lines[end_index]) > 1:
            current_line = f"{current_line[:-1]} {lines[end_index].lstrip()}"
        end_index += 1
    if len(current_line) > 1:
        result.append(current_line)



SUMMARY:
**Function Summary:**

`get_model_list`: Extracts and returns the model list from a README file.

**Purpose**: Finds the start and end markers of the model list in the specified file, reads between them, and returns the extracted list as a string.

**Key Implementation Details:**

* Reads the entire file into memory.
* Uses two prompts (`start_prompt` and `end_prompt`) to determine the start and end indices of the model list.
* Iterates through lines until it finds the end prompt, appending each line that starts with "1." (presumably a model name) or has additional content beyond a single character.
* Handles cases where the current line has more than one character after processing.

**Dependencies and Interfaces:**

* `os`: for file path manipulation
* `REPO_PATH` (not shown in this code snippet): likely an environment variable or global constant containing the repository root directory.


key: chroma:document
string_value:
CODE:

def convert_to_localized_md(model_list: str, localized_model_list: str, format_str: str) -> Tuple[bool, str]:
    """
    Compare the model list from the main README to the one in a localized README.

    Args:
        model_list (`str`): The model list in the main README.
        localized_model_list (`str`): The model list in one of the localized README.
        format_str (`str`):
            The template for a model entry in the localized README (look at the `format_model_list` in the entries of
            `LOCALIZED_READMES` for examples).

    Returns:
        `Tuple[bool, str]`: A tuple where the first value indicates if the READMEs match or not, and the second value
        is the correct localized README.
    """

    def _rep(match):
        title, model_link, paper_affiliations, paper_title_link, paper_authors, supplements = match.groups()
        return format_str.format(
            title=title,
            model_link=model_link,
            paper_affiliations=paper_affiliations,
            paper_title_link=paper_title_link,
            paper_authors=paper_authors,
            supplements=" " + supplements.strip() if len(supplements) != 0 else "",
        )

    # This regex captures metadata from an English model description, including model title, model link,
    # affiliations of the paper, title of the paper, authors of the paper, and supplemental data (see DistilBERT for
    # example).
    _re_capture_meta = re.compile(
        r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\* \(from ([^)]*)\)[^\[]*([^\)]*\)).*?by (.*?[A-Za-z\*]{2,}?)\. (.*)$"
    )
    # This regex is used to synchronize title link.
    _re_capture_title_link = re.compile(r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\*")
    # This regex is used to synchronize paper title and link.
    _re_capture_paper_link = re.compile(r" \[([^\]]*)\]\(([^\)]*)\)")

    if len(localized_model_list) == 0:
        localized_model_index = {}
    else:
        try:
            localized_model_index = {
                re.search(r"\*\*\[([^\]]*)", line).groups()[0]: line
                for line in localized_model_list.strip().split("\n")
            }
        except AttributeError:
            raise AttributeError("A model name in localized READMEs cannot be recognized.")

    model_keys = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in model_list.strip().split("\n")]

    # We exclude keys in localized README not in the main one.
    readmes_match = not any(k not in model_keys for k in localized_model_index)
    localized_model_index = {k: v for k, v in localized_model_index.items() if k in model_keys}

    for model in model_list.strip().split("\n"):
        title, model_link = _re_capture_title_link.search(model).groups()
        if title not in localized_model_index:
            readmes_match = False
            # Add an anchor white space behind a model description string for regex.
            # If metadata cannot be captured, the English version will be directly copied.
            localized_model_index[title] = _re_capture_meta.sub(_rep, model + " ")
        elif _re_fill_pattern.search(localized_model_index[title]) is not None:
            update = _re_capture_meta.sub(_rep, model + " ")
            if update != localized_model_index[title]:
                readmes_match = False
                localized_model_index[title] = update
        else:
            # Synchronize title link
            converted_model = _re_capture_title_link.sub(
                f"**[{title}]({model_link})**", localized_model_index[title], count=1
            )

            # Synchronize paper title and its link (if found)
            paper_title_link = _re_capture_paper_link.search(model)
            if paper_title_link is not None:
                paper_title, paper_link = paper_title_link.groups()
                converted_model = _re_capture_paper_link.sub(
                    f" [{paper_title}]({paper_link})", converted_model, count=1
                )

            if converted_model != localized_model_index[title]:
                readmes_match = False
                localized_model_index[title] = converted_model

    sorted_index = sorted(localized_model_index.items(), key=lambda x: x[0].lower())



SUMMARY:
**Function Summary**

`convert_to_localized_md`

Compares model list from main README to localized README and updates the latter with synchronized metadata.

**Main Purpose and Functionality**

This function compares two model lists, one from the main README and another from a localized README. It synchronizes metadata such as titles, model links, paper affiliations, and authors between the two lists.

**Key Implementation Details**

* Uses regular expressions to capture metadata from English model descriptions.
* Checks for matching models in both READMEs and updates the localized one if necessary.
* Synchronizes title link, paper title, and its link (if found) using regex patterns.

**Notable Dependencies or Requirements**

* `re` module for regular expression operations.
* `Tuple` type hint to indicate return value format.

Note: The original function has been preserved with the most important details. Note that functions like `format_model_list`, `LOCALIZED_READMES`, and `_re_capture_title_link` are not directly analyzed here as they appear to be referenced components instead of internal implementation details.


key: chroma:document
string_value:
CODE:

def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tuple[str, int, int, List[str]]:
    """
    Find the text in a file between two prompts.

    Args:
        filename (`str`): The name of the file to look into.
        start_prompt (`str`): The string to look for that introduces the content looked for.
        end_prompt (`str`): The string to look for that ends the content looked for.

    Returns:
        Tuple[str, int, int, List[str]]: The content between the two prompts, the index of the start line in the
        original file, the index of the end line in the original file and the list of lines of that file.
    """
    with open(filename, "r", encoding="utf-8", newline="\n") as f:
        lines = f.readlines()
    # Find the start prompt.
    start_index = 0
    while not lines[start_index].startswith(start_prompt):
        start_index += 1
    start_index += 1

    end_index = start_index
    while not lines[end_index].startswith(end_prompt):
        end_index += 1
    end_index -= 1

    while len(lines[start_index]) <= 1:
        start_index += 1
    while len(lines[end_index]) <= 1:
        end_index -= 1
    end_index += 1


SUMMARY:
**Function Summary:**

`_find_text_in_file` function finds text in a file between two prompts.
```python
import typing as t
from typing import Tuple, List

def _find_text_in_file(filename: str, start_prompt: str, end_prompt: str) -> Tuple[str, int, int, List[str]]:
    """
    Args:
        filename (str): File to search in.
        start_prompt (str): String introducing the content searched for.
        end_prompt (str): String marking the end of the content searched for.

    Returns:
        A tuple containing:
            - The extracted text.
            - The index of the start line in the original file.
            - The index of the end line in the original file.
            - The list of all lines in the file.
    """
```

**Key Implementation Details:**

* Finds start and end indices by searching for prompt strings.
* Handling cases where prompts may be on empty lines or follow multiple empty lines.

**Notable Dependencies and Requirements:**

* `open` function from Python's standard library for reading files.
* `re` module is not explicitly imported, but it might be implicitly used through string methods.


key: chroma:document
string_value:
CODE:

def get_runner_status(target_runners, token):
    offline_runners = []

    cmd = (
        f'curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer {token}"'
        " https://api.github.com/repos/huggingface/transformers/actions/runners"
    )
    output = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE)
    o = output.stdout.decode("utf-8")
    status = json.loads(o)

    runners = status["runners"]
    for runner in runners:
        if runner["name"] in target_runners:
            if runner["status"] == "offline":
                offline_runners.append(runner)

    # save the result so we can report them on Slack
    with open("offline_runners.txt", "w") as fp:
        fp.write(json.dumps(offline_runners))

    if len(offline_runners) > 0:
        failed = "\n".join([x["name"] for x in offline_runners])


SUMMARY:
**Code Summary**

### Function Purpose

`get_runner_status` retrieves the status of GitHub Actions runners from the GitHub API.

### Key Implementation Details

* Fetches runner status using `curl` and Parses the response with JSON
* Compares runner names against `target_runners` list to retrieve offline runnersonly
* Saves result to `offline_runners.txt`

### Notable Dependencies/Requirements

* `github-api`: implicit, accessed via `https://api.github.com/repos/huggingface/transformers/actions/runners`
* `json` and `subprocess` modules for standard I/O and execution
* `Authorization` token for authentication (Bearer `token`)


key: chroma:document
string_value:
CODE:

def check_flash_support_list():
    with open(os.path.join(REPO_PATH, "docs/source/en/perf_infer_gpu_one.md"), "r") as f:
        doctext = f.read()

        doctext = doctext.split("FlashAttention-2 is currently supported for the following architectures:")[1]
        doctext = doctext.split("You can request to add FlashAttention-2 support")[0]

    patterns = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_*.py"))
    patterns_tf = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_tf_*.py"))
    patterns_flax = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_flax_*.py"))
    patterns = list(set(patterns) - set(patterns_tf) - set(patterns_flax))
    archs_supporting_fa2 = []
    for filename in patterns:
        with open(filename, "r") as f:
            text = f.read()

            if "_supports_flash_attn_2 = True" in text:
                model_name = os.path.basename(filename).replace(".py", "").replace("modeling_", "")
                archs_supporting_fa2.append(model_name)

    for arch in archs_supporting_fa2:
        if arch not in doctext:
            raise ValueError(
                f"{arch} should be in listed in the flash attention documentation but is not. Please update the documentation."


SUMMARY:
**Function Summary: `check_flash_support_list`**

Purpose:
This function checks if FlashAttention-2 (FA2) support is included for various architectures based on documentation and code patterns.

Main Functionality:

1. Reads documentation text to extract supported architectures.
2. Searches for model files with FA2 support in `modeling*` pattern and initializes a list of supported architectures (`archs_supporting_fa2`).
3. Compares the extracted supported architectures against the ones listed in FA2 documentation and raises a ValueError if any mismatch.

Key Implementation Details:

- Uses glob to search for files under `src/transformers/models` with models containing "_supports_flash_attn_2 = True".
- Utilizes string splitting to extract relevant text from documentation files.
- Compares lists of supported architectures using set operations.

Notable Dependencies and Requirements:
- `os.path.join` and `open` functions for file system interactions
- `glob` function for pattern matching
- `REPO_PATH` constant referencing the repository's root path
- External dependency: FlashAttention documentation (`perf_infer_gpu_one.md`) and model files containing "_supports_flash_attn_2 = True".


key: chroma:document
string_value:
CODE:

def check_sdpa_support_list():
    with open(os.path.join(REPO_PATH, "docs/source/en/perf_infer_gpu_one.md"), "r") as f:
        doctext = f.read()

        doctext = doctext.split(
            "For now, Transformers supports SDPA inference and training for the following architectures:"
        )[1]
        doctext = doctext.split("Note that FlashAttention can only be used for models using the")[0]
        doctext = doctext.lower()

    patterns = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_*.py"))
    patterns_tf = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_tf_*.py"))
    patterns_flax = glob(os.path.join(REPO_PATH, "src/transformers/models/**/modeling_flax_*.py"))
    patterns = list(set(patterns) - set(patterns_tf) - set(patterns_flax))
    archs_supporting_sdpa = []
    for filename in patterns:
        with open(filename, "r") as f:
            text = f.read()

            if "_supports_sdpa = True" in text:
                model_name = os.path.basename(filename).replace(".py", "").replace("modeling_", "")
                archs_supporting_sdpa.append(model_name)

    for arch in archs_supporting_sdpa:
        if not any(term in doctext for term in [arch, arch.replace("_", "-"), arch.replace("_", " ")]):
            raise ValueError(
                f"{arch} should be in listed in the SDPA documentation but is not. Please update the documentation."


SUMMARY:
**Functionality Summary:**

The `check_sdpa_support_list` function checks if various Transformer models support SDPA (Software Distributed Performance Architecture) inference and training.

### Main Purpose:

1. Retrieves documentation text from a file (`docs/source/en/perf_infer_gpu_one.md`) to determine which architectures are supported by SDPA.
2. Iterates through model files in the `src/transformers/models` directory and extracts model names that contain the line `_supports_sdpa = True`.
3. Verifies if each detected model is listed in the retrieved documentation text, raising a ValueError if any are not found.

### Key Implementation Details:**

* The function uses regular expressions to extract specific lines from the documentation text.
* It utilizes glob to find model files matching certain naming patterns (`modeling_*.py`) and excludes files using `FlashAttention`.
* Model names are appended to a list (`archs_supporting_sdpa`) based on the presence of the `_supports_sdpa = True` line.

### Notable Dependencies and Requirements:**

* The function relies on the following external dependencies:
	+ `os.path.join()` for path manipulation
	+ `fnmatch` for glob pattern matching (not explicitly required, as Python's built-in `glob` module is used)
	+ `re` for regular expression operations (not explicitly required, but implied by string splitting and searching)

### Contextual Considerations:

This function appears to be part of a automated documentation checking system, verifying the support list of various Transformer models against actual implementation.


key: chroma:document
string_value:
CODE:

def handle_test_results(test_results):
    expressions = test_results.split(" ")

    failed = 0
    success = 0

    # When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
    # When it is too long, those signs are not present.
    time_spent = expressions[-2] if "=" in expressions[-1] else expressions[-1]

    for i, expression in enumerate(expressions):
        if "failed" in expression:
            failed += int(expressions[i - 1])
        if "passed" in expression:
            success += int(expressions[i - 1])



SUMMARY:
**Summary:**

`handle_test_results` function processes test results by:

* Splitting into individual expressions containing "failed" and "passed"
* Calculating total failed and success tests count
* Extracting time spent in the output string, assuming it's surrounded by "= =" for short outputs

**Implementation Details:**

* Iterates through expressions using `enumerate`
* Uses exact method signaturs to access values: `int(expression[i - 1])` 

**Dependencies/Requirements:**

* Presumes test results are comma-separated strings
* Expects output string to contain a specific format (`"{value} = {expression}"`)
* No external dependencies listed, but assumes `test_results` input is valid.


key: chroma:document
string_value:
CODE:

def handle_stacktraces(test_results):
    # These files should follow the following architecture:
    # === FAILURES ===
    # <path>:<line>: Error ...
    # <path>:<line>: Error ...
    # <empty line>

    total_stacktraces = test_results.split("\n")[1:-1]
    stacktraces = []
    for stacktrace in total_stacktraces:
        try:
            line = stacktrace[: stacktrace.index(" ")].split(":")[-2]
            error_message = stacktrace[stacktrace.index(" ") :]

            stacktraces.append(f"(line {line}) {error_message}")
        except Exception:
            stacktraces.append("Cannot retrieve error message.")



SUMMARY:
**Function Summary**

`handle_stacktraces`: Processes test results to extract and format error messages from stacktraces.

### Functionality

* Takes `test_results` as input
* Splits input into lines, excluding the first and last (header/footer)
* Iterates through each line, extracting:
	+ Line number from path element before first space character
	+ Error message from remaining path elements 
* Returns a list of formatted error messages

### Key Implementation Details

* `total_stacktraces` is extracted using `split("\n")`
* Uses indexing to extract line and error parts of each stacktrace
* Catches exceptions during processing, appending "Cannot retrieve error message." if an error occurs

### Notable Dependencies

* No external dependencies referenced
* Assumes input test results are structured according to specified architecture


key: chroma:document
string_value:
CODE:

def dicts_to_sum(objects: Union[Dict[str, Dict], List[dict]]):
    if isinstance(objects, dict):
        lists = objects.values()
    else:
        lists = objects

    # Convert each dictionary to counter
    counters = map(collections.Counter, lists)
    # Sum all the counters


SUMMARY:
**Function Summary**

`dicts_to_sum` function:

* Takes a single `objects` parameter with two acceptable inputs:
	+ A dictionary of dictionaries (`Dict[str, Dict]`)
	+ A list of dictionaries (`List[dict]`)
* If input is a dictionary, it extracts the list of inner dictionaries.
* It uses `Counter` from `collections` to count occurrences of each key-value pair in each dictionary and then maps this counter across all lists using `functools.reduce` with `operator.add`, summing up the results.

**Key Implementation Details:**

* Uses `collections.Counter` for efficient counting
* Applies a cumulative reduction using `functools.reduce` and `operator.add`

**Notable Dependencies/R Requirements:**

* `unico` (not 'Union', assumed typo, corrected to `Union`, actual is not present in this code snippet)
    * No explicit dependencies, but it utilizes:
        + `collections.Counter`
        + `functools.reduce`
        + `operator.add`
        + Assume a dictionary `Dict[str, Dict]` for conversion from other list format or use `List[dict]` as stated in the documentation


key: chroma:document
string_value:
CODE:

class Message:
    def __init__(
        self,
        title: str,
        ci_title: str,
        model_results: Dict,
        additional_results: Dict,
        selected_warnings: List = None,
        prev_ci_artifacts=None,
    ):
        self.title = title
        self.ci_title = ci_title

        # Failures and success of the modeling tests
        self.n_model_success = sum(r["success"] for r in model_results.values())
        self.n_model_single_gpu_failures = sum(dicts_to_sum(r["failed"])["single"] for r in model_results.values())
        self.n_model_multi_gpu_failures = sum(dicts_to_sum(r["failed"])["multi"] for r in model_results.values())

        # Some suites do not have a distinction between single and multi GPU.
        self.n_model_unknown_failures = sum(dicts_to_sum(r["failed"])["unclassified"] for r in model_results.values())
        self.n_model_failures = (
            self.n_model_single_gpu_failures + self.n_model_multi_gpu_failures + self.n_model_unknown_failures
        )

        # Failures and success of the additional tests
        self.n_additional_success = sum(r["success"] for r in additional_results.values())

        if len(additional_results) > 0:
            # `dicts_to_sum` uses `dicts_to_sum` which requires a non empty dictionary. Let's just add an empty entry.
            all_additional_failures = dicts_to_sum([r["failed"] for r in additional_results.values()])
            self.n_additional_single_gpu_failures = all_additional_failures["single"]
            self.n_additional_multi_gpu_failures = all_additional_failures["multi"]
            self.n_additional_unknown_gpu_failures = all_additional_failures["unclassified"]
        else:
            self.n_additional_single_gpu_failures = 0
            self.n_additional_multi_gpu_failures = 0
            self.n_additional_unknown_gpu_failures = 0

        self.n_additional_failures = (
            self.n_additional_single_gpu_failures
            + self.n_additional_multi_gpu_failures
            + self.n_additional_unknown_gpu_failures
        )

        # Results
        self.n_failures = self.n_model_failures + self.n_additional_failures
        self.n_success = self.n_model_success + self.n_additional_success
        self.n_tests = self.n_failures + self.n_success

        self.model_results = model_results
        self.additional_results = additional_results

        self.thread_ts = None

        if selected_warnings is None:
            selected_warnings = []
        self.selected_warnings = selected_warnings

        self.prev_ci_artifacts = prev_ci_artifacts

    @property
    def time(self) -> str:
        all_results = [*self.model_results.values(), *self.additional_results.values()]
        time_spent = [r["time_spent"].split(", ")[0] for r in all_results if len(r["time_spent"])]
        total_secs = 0

        for time in time_spent:
            time_parts = time.split(":")

            # Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
            if len(time_parts) == 1:
                time_parts = [0, 0, time_parts[0]]

            hours, minutes, seconds = int(time_parts[0]), int(time_parts[1]), float(time_parts[2])
            total_secs += hours * 3600 + minutes * 60 + seconds

        hours, minutes, seconds = total_secs // 3600, (total_secs % 3600) // 60, total_secs % 60
        return f"{int(hours)}h{int(minutes)}m{int(seconds)}s"

    @property
    def header(self) -> Dict:
        return {"type": "header", "text": {"type": "plain_text", "text": self.title}}

    @property
    def ci_title_section(self) -> Dict:
        return {"type": "section", "text": {"type": "mrkdwn", "text": self.ci_title}}

    @property
    def no_failures(self) -> Dict:
        return {
            "type": "section",
            "text": {
                "type": "plain_text",
                "text": f"🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.",
                "emoji": True,
            },
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
            },
        }

    @property
    def failures(self) -> Dict:
        return {
            "type": "section",
            "text": {
                "type": "plain_text",
                "text": (
                    f"There were {self.n_failures} failures, out of {self.n_tests} tests.\n"
                    f"Number of model failures: {self.n_model_failures}.\n"
                    f"The suite ran in {self.time}."
                ),
                "emoji": True,
            },
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
            },
        }

    @property
    def warnings(self) -> Dict:
        # If something goes wrong, let's avoid the CI report failing to be sent.
        button_text = "Check warnings (Link not found)"
        # Use the workflow run link
        job_link = f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}"

        for job in github_actions_jobs:
            if "Extract warnings in CI artifacts" in job["name"] and job["conclusion"] == "success":
                button_text = "Check warnings"
                # Use the actual job link
                job_link = job["html_url"]
                break

        huggingface_hub_warnings = [x for x in self.selected_warnings if "huggingface_hub" in x]
        text = f"There are {len(self.selected_warnings)} warnings being selected."
        text += f"\n{len(huggingface_hub_warnings)} of them are from `huggingface_hub`."

        return {
            "type": "section",
            "text": {
                "type": "plain_text",
                "text": text,
                "emoji": True,
            },
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": button_text, "emoji": True},
                "url": job_link,
            },
        }

    @staticmethod
    def get_device_report(report, rjust=6):
        if "single" in report and "multi" in report:
            return f"{str(report['single']).rjust(rjust)} | {str(report['multi']).rjust(rjust)} | "
        elif "single" in report:
            return f"{str(report['single']).rjust(rjust)} | {'0'.rjust(rjust)} | "
        elif "multi" in report:
            return f"{'0'.rjust(rjust)} | {str(report['multi']).rjust(rjust)} | "

    @property
    def category_failures(self) -> Dict:
        model_failures = [v["failed"] for v in self.model_results.values()]

        category_failures = {}

        for model_failure in model_failures:
            for key, value in model_failure.items():
                if key not in category_failures:
                    category_failures[key] = dict(value)
                else:
                    category_failures[key]["unclassified"] += value["unclassified"]
                    category_failures[key]["single"] += value["single"]
                    category_failures[key]["multi"] += value["multi"]

        individual_reports = []
        for key, value in category_failures.items():
            device_report = self.get_device_report(value)

            if sum(value.values()):
                if device_report:
                    individual_reports.append(f"{device_report}{key}")
                else:
                    individual_reports.append(key)

        header = "Single |  Multi | Category\n"
        category_failures_report = prepare_reports(
            title="The following modeling categories had failures", header=header, reports=individual_reports
        )

        return {"type": "section", "text": {"type": "mrkdwn", "text": category_failures_report}}

    def compute_diff_for_failure_reports(self, curr_failure_report, prev_failure_report):  # noqa
        # Remove the leading and training parts that don't contain failure count information.
        model_failures = curr_failure_report.split("\n")[3:-2]
        prev_model_failures = prev_failure_report.split("\n")[3:-2]
        entries_changed = set(model_failures).difference(prev_model_failures)

        prev_map = {}
        for f in prev_model_failures:
            items = [x.strip() for x in f.split("| ")]
            prev_map[items[-1]] = [int(x) for x in items[:-1]]

        curr_map = {}
        for f in entries_changed:
            items = [x.strip() for x in f.split("| ")]
            curr_map[items[-1]] = [int(x) for x in items[:-1]]

        diff_map = {}
        for k, v in curr_map.items():
            if k not in prev_map:
                diff_map[k] = v
            else:
                diff = [x - y for x, y in zip(v, prev_map[k])]
                if max(diff) > 0:
                    diff_map[k] = diff

        entries_changed = []
        for model_name, diff_values in diff_map.items():
            diff = [str(x) for x in diff_values]
            diff = [f"+{x}" if (x != "0" and not x.startswith("-")) else x for x in diff]
            diff = [x.rjust(9) for x in diff]
            device_report = " | ".join(diff) + " | "
            report = f"{device_report}{model_name}"
            entries_changed.append(report)
        entries_changed = sorted(entries_changed, key=lambda s: s.split("| ")[-1])

        return entries_changed

    @property
    def model_failures(self) -> List[Dict]:
        # Obtain per-model failures
        def per_model_sum(model_category_dict):
            return dicts_to_sum(model_category_dict["failed"].values())

        failures = {}
        non_model_failures = {
            k: per_model_sum(v) for k, v in self.model_results.items() if sum(per_model_sum(v).values())
        }

        for k, v in self.model_results.items():
            if k in NON_MODEL_TEST_MODULES:
                pass

            if sum(per_model_sum(v).values()):
                dict_failed = dict(v["failed"])
                pytorch_specific_failures = dict_failed.pop("PyTorch")
                tensorflow_specific_failures = dict_failed.pop("TensorFlow")
                other_failures = dicts_to_sum(dict_failed.values())

                failures[k] = {
                    "PyTorch": pytorch_specific_failures,
                    "TensorFlow": tensorflow_specific_failures,
                    "other": other_failures,
                }

        model_reports = []
        other_module_reports = []

        for key, value in non_model_failures.items():
            if key in NON_MODEL_TEST_MODULES:
                device_report = self.get_device_report(value)

                if sum(value.values()):
                    if device_report:
                        report = f"{device_report}{key}"
                    else:
                        report = key

                    other_module_reports.append(report)

        for key, value in failures.items():
            device_report_values = [
                value["PyTorch"]["single"],
                value["PyTorch"]["multi"],
                value["TensorFlow"]["single"],
                value["TensorFlow"]["multi"],
                sum(value["other"].values()),
            ]

            if sum(device_report_values):
                device_report = " | ".join([str(x).rjust(9) for x in device_report_values]) + " | "
                report = f"{device_report}{key}"

                model_reports.append(report)

        # (Possibly truncated) reports for the current workflow run - to be sent to Slack channels
        model_header = "Single PT |  Multi PT | Single TF |  Multi TF |     Other | Category\n"
        sorted_model_reports = sorted(model_reports, key=lambda s: s.split("| ")[-1])
        model_failures_report = prepare_reports(
            title="These following model modules had failures", header=model_header, reports=sorted_model_reports
        )

        module_header = "Single |  Multi | Category\n"
        sorted_module_reports = sorted(other_module_reports, key=lambda s: s.split("| ")[-1])
        module_failures_report = prepare_reports(
            title="The following non-model modules had failures", header=module_header, reports=sorted_module_reports
        )

        # To be sent to Slack channels
        model_failure_sections = [
            {"type": "section", "text": {"type": "mrkdwn", "text": model_failures_report}},
            {"type": "section", "text": {"type": "mrkdwn", "text": module_failures_report}},
        ]

        # Save the complete (i.e. no truncation) failure tables (of the current workflow run)
        # (to be uploaded as artifacts)

        model_failures_report = prepare_reports(
            title="These following model modules had failures",
            header=model_header,
            reports=sorted_model_reports,
            to_truncate=False,
        )
        file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/model_failures_report.txt")
        with open(file_path, "w", encoding="UTF-8") as fp:
            fp.write(model_failures_report)

        module_failures_report = prepare_reports(
            title="The following non-model modules had failures",
            header=module_header,
            reports=sorted_module_reports,
            to_truncate=False,
        )
        file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/module_failures_report.txt")
        with open(file_path, "w", encoding="UTF-8") as fp:
            fp.write(module_failures_report)

        if self.prev_ci_artifacts is not None:
            # if the last run produces artifact named `ci_results_{job_name}`
            if (
                f"ci_results_{job_name}" in self.prev_ci_artifacts
                and "model_failures_report.txt" in self.prev_ci_artifacts[f"ci_results_{job_name}"]
            ):
                # Compute the difference of the previous/current (model failure) table
                prev_model_failures = self.prev_ci_artifacts[f"ci_results_{job_name}"]["model_failures_report.txt"]
                entries_changed = self.compute_diff_for_failure_reports(model_failures_report, prev_model_failures)
                if len(entries_changed) > 0:
                    # Save the complete difference
                    diff_report = prepare_reports(
                        title="Changed model modules failures",
                        header=model_header,
                        reports=entries_changed,
                        to_truncate=False,
                    )
                    file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/changed_model_failures_report.txt")
                    with open(file_path, "w", encoding="UTF-8") as fp:
                        fp.write(diff_report)

                    # To be sent to Slack channels
                    diff_report = prepare_reports(
                        title="*Changed model modules failures*",
                        header=model_header,
                        reports=entries_changed,
                    )
                    model_failure_sections.append(
                        {"type": "section", "text": {"type": "mrkdwn", "text": diff_report}},
                    )

        return model_failure_sections

    @property
    def additional_failures(self) -> Dict:
        failures = {k: v["failed"] for k, v in self.additional_results.items()}
        errors = {k: v["error"] for k, v in self.additional_results.items()}

        individual_reports = []
        for key, value in failures.items():
            device_report = self.get_device_report(value)

            if sum(value.values()) or errors[key]:
                report = f"{key}"
                if errors[key]:
                    report = f"[Errored out] {report}"
                if device_report:
                    report = f"{device_report}{report}"

                individual_reports.append(report)

        header = "Single |  Multi | Category\n"
        failures_report = prepare_reports(
            title="The following non-modeling tests had failures", header=header, reports=individual_reports
        )

        return {"type": "section", "text": {"type": "mrkdwn", "text": failures_report}}

    @property
    def payload(self) -> str:
        blocks = [self.header]

        if self.ci_title:
            blocks.append(self.ci_title_section)

        if self.n_model_failures > 0 or self.n_additional_failures > 0:
            blocks.append(self.failures)

        if self.n_model_failures > 0:
            blocks.append(self.category_failures)
            for block in self.model_failures:
                if block["text"]["text"]:
                    blocks.append(block)

        if self.n_additional_failures > 0:
            blocks.append(self.additional_failures)

        if self.n_model_failures == 0 and self.n_additional_failures == 0:
            blocks.append(self.no_failures)

        if len(self.selected_warnings) > 0:
            blocks.append(self.warnings)

        new_failure_blocks = self.get_new_model_failure_blocks(with_header=False)
        if len(new_failure_blocks) > 0:
            blocks.extend(new_failure_blocks)

        # To save the list of new model failures
        extra_blocks = self.get_new_model_failure_blocks(to_truncate=False)
        if extra_blocks:
            failure_text = extra_blocks[-1]["text"]["text"]
            file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/new_model_failures.txt")
            with open(file_path, "w", encoding="UTF-8") as fp:
                fp.write(failure_text)

            # upload results to Hub dataset
            file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/new_model_failures.txt")
            commit_info = api.upload_file(
                path_or_fileobj=file_path,
                path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.txt",
                repo_id="hf-internal-testing/transformers_daily_ci",
                repo_type="dataset",
                token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
            )
            url = f"https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/{commit_info.oid}/{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.txt"

            # extra processing to save to json format
            new_failed_tests = {}
            for line in failure_text.split():
                if "https://github.com/huggingface/transformers/actions/runs" in line:
                    pattern = r"<(https://github.com/huggingface/transformers/actions/runs/.+?/job/.+?)\|(.+?)>"
                    items = re.findall(pattern, line)
                elif "tests/models/" in line:
                    model = line.split("/")[2]
                    if model not in new_failed_tests:
                        new_failed_tests[model] = {"single-gpu": [], "multi-gpu": []}
                    for url, device in items:
                        new_failed_tests[model][f"{device}-gpu"].append(line)
            file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/new_model_failures.json")
            with open(file_path, "w", encoding="UTF-8") as fp:
                json.dump(new_failed_tests, fp, ensure_ascii=False, indent=4)

            # upload results to Hub dataset
            file_path = os.path.join(os.getcwd(), f"ci_results_{job_name}/new_model_failures.json")
            _ = api.upload_file(
                path_or_fileobj=file_path,
                path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.json",
                repo_id="hf-internal-testing/transformers_daily_ci",
                repo_type="dataset",
                token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
            )

            block = {
                "type": "section",
                "text": {
                    "type": "plain_text",
                    "text": " ",
                },
                "accessory": {
                    "type": "button",
                    "text": {"type": "plain_text", "text": "Check New model failures"},
                    "url": url,
                },
            }
            blocks.append(block)

        return json.dumps(blocks)

    @staticmethod
    def error_out(title, ci_title="", runner_not_available=False, runner_failed=False, setup_failed=False):
        blocks = []
        title_block = {"type": "header", "text": {"type": "plain_text", "text": title}}
        blocks.append(title_block)

        if ci_title:
            ci_title_block = {"type": "section", "text": {"type": "mrkdwn", "text": ci_title}}
            blocks.append(ci_title_block)

        offline_runners = []
        if runner_not_available:
            text = "💔 CI runners are not available! Tests are not run. 😭"
            result = os.environ.get("OFFLINE_RUNNERS")
            if result is not None:
                offline_runners = json.loads(result)
        elif runner_failed:
            text = "💔 CI runners have problems! Tests are not run. 😭"
        elif setup_failed:
            text = "💔 Setup job failed. Tests are not run. 😭"
        else:
            text = "💔 There was an issue running the tests. 😭"

        error_block_1 = {
            "type": "header",
            "text": {
                "type": "plain_text",
                "text": text,
            },
        }

        text = ""
        if len(offline_runners) > 0:
            text = "\n  • " + "\n  • ".join(offline_runners)
            text = f"The following runners are offline:\n{text}\n\n"
        text += "🙏 Let's fix it ASAP! 🙏"

        error_block_2 = {
            "type": "section",
            "text": {
                "type": "plain_text",
                "text": text,
            },
            "accessory": {
                "type": "button",
                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                "url": f"https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
            },
        }
        blocks.extend([error_block_1, error_block_2])

        payload = json.dumps(blocks)

        print("Sending the following payload")
        print(json.dumps({"blocks": blocks}))

        client.chat_postMessage(
            channel=SLACK_REPORT_CHANNEL_ID,
            text=text,
            blocks=payload,
        )

    def post(self):
        payload = self.payload
        print("Sending the following payload")
        print(json.dumps({"blocks": json.loads(payload)}))

        text = f"{self.n_failures} failures out of {self.n_tests} tests," if self.n_failures else "All tests passed."

        self.thread_ts = client.chat_postMessage(
            channel=SLACK_REPORT_CHANNEL_ID,
            blocks=payload,
            text=text,
        )

    def get_reply_blocks(self, job_name, job_result, failures, device, text):
        """
        failures: A list with elements of the form {"line": full test name, "trace": error trace}
        """
        # `text` must be less than 3001 characters in Slack SDK
        # keep some room for adding "[Truncated]" when necessary
        MAX_ERROR_TEXT = 3000 - len("[Truncated]")

        failure_text = ""
        for idx, error in enumerate(failures):
            new_text = failure_text + f'*{error["line"]}*\n_{error["trace"]}_\n\n'
            if len(new_text) > MAX_ERROR_TEXT:
                # `failure_text` here has length <= 3000
                failure_text = failure_text + "[Truncated]"
                break
            # `failure_text` here has length <= MAX_ERROR_TEXT
            failure_text = new_text

        title = job_name
        if device is not None:
            title += f" ({device}-gpu)"

        content = {"type": "section", "text": {"type": "mrkdwn", "text": text}}

        # TODO: Make sure we always have a valid job link (or at least a way not to break the report sending)
        # Currently we get the device from a job's artifact name.
        # If a device is found, the job name should contain the device type, for example, `XXX (single-gpu)`.
        # This could be done by adding `machine_type` in a job's `strategy`.
        # (If `job_result["job_link"][device]` is `None`, we get an error: `... [ERROR] must provide a string ...`)
        if job_result["job_link"] is not None and job_result["job_link"][device] is not None:
            content["accessory"] = {
                "type": "button",
                "text": {"type": "plain_text", "text": "GitHub Action job", "emoji": True},
                "url": job_result["job_link"][device],
            }

        return [
            {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
            content,
            {"type": "section", "text": {"type": "mrkdwn", "text": failure_text}},
        ]

    def get_new_model_failure_blocks(self, with_header=True, to_truncate=True):
        if self.prev_ci_artifacts is None:
            return []

        sorted_dict = sorted(self.model_results.items(), key=lambda t: t[0])

        prev_model_results = {}
        if (
            f"ci_results_{job_name}" in self.prev_ci_artifacts
            and "model_results.json" in self.prev_ci_artifacts[f"ci_results_{job_name}"]
        ):
            prev_model_results = json.loads(self.prev_ci_artifacts[f"ci_results_{job_name}"]["model_results.json"])

        all_failure_lines = {}
        for job, job_result in sorted_dict:
            if len(job_result["failures"]):
                devices = sorted(job_result["failures"].keys(), reverse=True)
                for device in devices:
                    failures = job_result["failures"][device]
                    prev_error_lines = {}
                    if job in prev_model_results and device in prev_model_results[job]["failures"]:
                        prev_error_lines = {error["line"] for error in prev_model_results[job]["failures"][device]}

                    url = None
                    if job_result["job_link"] is not None and job_result["job_link"][device] is not None:
                        url = job_result["job_link"][device]

                    for idx, error in enumerate(failures):
                        if error["line"] in prev_error_lines:
                            continue

                        new_text = f'{error["line"]}\n\n'

                        if new_text not in all_failure_lines:
                            all_failure_lines[new_text] = []

                        all_failure_lines[new_text].append(f"<{url}|{device}>" if url is not None else device)

        MAX_ERROR_TEXT = 3000 - len("[Truncated]") - len("```New model failures```\n\n")
        if not to_truncate:
            MAX_ERROR_TEXT = float("inf")
        failure_text = ""
        for line, devices in all_failure_lines.items():
            new_text = failure_text + f"{'|'.join(devices)} gpu\n{line}"
            if len(new_text) > MAX_ERROR_TEXT:
                # `failure_text` here has length <= 3000
                failure_text = failure_text + "[Truncated]"
                break
            # `failure_text` here has length <= MAX_ERROR_TEXT
            failure_text = new_text

        blocks = []
        if failure_text:
            if with_header:
                blocks.append(
                    {"type": "header", "text": {"type": "plain_text", "text": "New model failures", "emoji": True}}
                )
            else:
                failure_text = f"*New model failures*\n\n{failure_text}"
            blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": failure_text}})

        return blocks

    def post_reply(self):
        if self.thread_ts is None:
            raise ValueError("Can only post reply if a post has been made.")

        sorted_dict = sorted(self.model_results.items(), key=lambda t: t[0])
        for job, job_result in sorted_dict:
            if len(job_result["failures"]):
                for device, failures in job_result["failures"].items():
                    text = "\n".join(
                        sorted([f"*{k}*: {v[device]}" for k, v in job_result["failed"].items() if v[device]])
                    )

                    blocks = self.get_reply_blocks(job, job_result, failures, device, text=text)

                    print("Sending the following reply")
                    print(json.dumps({"blocks": blocks}))

                    client.chat_postMessage(
                        channel=SLACK_REPORT_CHANNEL_ID,
                        text=f"Results for {job}",
                        blocks=blocks,
                        thread_ts=self.thread_ts["ts"],
                    )

                    time.sleep(1)

        for job, job_result in self.additional_results.items():
            if len(job_result["failures"]):
                for device, failures in job_result["failures"].items():
                    blocks = self.get_reply_blocks(
                        job,
                        job_result,
                        failures,
                        device,
                        text=f'Number of failures: {job_result["failed"][device]}',
                    )

                    print("Sending the following reply")
                    print(json.dumps({"blocks": blocks}))

                    client.chat_postMessage(
                        channel=SLACK_REPORT_CHANNEL_ID,
                        text=f"Results for {job}",
                        blocks=blocks,
                        thread_ts=self.thread_ts["ts"],
                    )

                    time.sleep(1)

        blocks = self.get_new_model_failure_blocks()
        if blocks:
            print("Sending the following reply")
            print(json.dumps({"blocks": blocks}))

            client.chat_postMessage(
                channel=SLACK_REPORT_CHANNEL_ID,
                text="Results for new failures",
                blocks=blocks,
                thread_ts=self.thread_ts["ts"],
            )



SUMMARY:
The provided code is a part of a Slackbot designed to report test results from machine learning jobs. The bot fetches the results from a database and formats them into threads that are sent as separate messages on Slack.

Key Features:

*   **Results Reporting:** The bot fetches and formats test results, which can be obtained from a database, into thread-like structures suitable for sending to Slack.
*   **Timing Mechanism:** There is a one-second delay between each post operation, which implies multiple posts per minute, given the rate of results and other operations like sleeping between the posts.
*   **Error Handling:** Missing results, CI run artifacts being None, or missing models' names are error conditions that need to be solved. Proper exception handling would greatly enhance this part's robustness.

Improvement Suggestions:

1.  Add Exception Handling:
    *   Consider raising exceptions instead of silently failing on certain operations like fetching results where `ci_results_{job_name}` does not exist.
2.  Optimize Post Operations:
    *   Currently, you're sending a separate post for each model fail and its failure rate. This might be inefficient due to sending large blocks or multiple posts. Simplify your strategy if possible.

Here's an example implementation of error handling with exceptions:

```python
try:
    # your operations here...
except Exception as e:
    print(f"An error occurred: {e}")
```

Remember that adding too many checks without balancing the code readability can make it difficult to understand and lead to better design choices. So pick wisely where you add exception handling to improve robustness.


key: chroma:document
string_value:
CODE:

def retrieve_artifact(artifact_path: str, gpu: Optional[str]):
    if gpu not in [None, "single", "multi"]:
        raise ValueError(f"Invalid GPU for artifact. Passed GPU: `{gpu}`.")

    _artifact = {}

    if os.path.exists(artifact_path):
        files = os.listdir(artifact_path)
        for file in files:
            try:
                with open(os.path.join(artifact_path, file)) as f:
                    _artifact[file.split(".")[0]] = f.read()
            except UnicodeDecodeError as e:
                raise ValueError(f"Could not open {os.path.join(artifact_path, file)}.") from e



SUMMARY:
**Function Summary**

`retrieve_artifact`: Retrieves artifacts from a specified path.

**Purpose and Functionality**
Extracts artifact files from the given `artifact_path`, including file contents in decoded format. Handles invalid GPU configurations and non-existent paths.

**Key Implementation Details**

* Checks valid GPU configurations (`single`, `multi`, or `None`).
* Uses `os.listdir` to get a list of files in the specified path.
* Iterates through each file, attempts to open and read its contents using `with` statement.
* Raises error on non-existent paths and fails to decode Unicode characters.

**Notable Dependencies and Requirements**

* `[os]` module for interacting with the file system.
* `Optional[str]` type hint for indicating that `gpu` can be `None`.
* `Unicode Decode Error` handling raised by Python.


key: chroma:document
string_value:
CODE:

def retrieve_available_artifacts():
    class Artifact:
        def __init__(self, name: str, single_gpu: bool = False, multi_gpu: bool = False):
            self.name = name
            self.single_gpu = single_gpu
            self.multi_gpu = multi_gpu
            self.paths = []

        def __str__(self):
            return self.name

        def add_path(self, path: str, gpu: str = None):
            self.paths.append({"name": self.name, "path": path, "gpu": gpu})

    _available_artifacts: Dict[str, Artifact] = {}

    directories = filter(os.path.isdir, os.listdir())
    for directory in directories:
        artifact_name = directory

        name_parts = artifact_name.split("_postfix_")
        if len(name_parts) > 1:
            artifact_name = name_parts[0]

        if artifact_name.startswith("single-gpu"):
            artifact_name = artifact_name[len("single-gpu") + 1 :]

            if artifact_name in _available_artifacts:
                _available_artifacts[artifact_name].single_gpu = True
            else:
                _available_artifacts[artifact_name] = Artifact(artifact_name, single_gpu=True)

            _available_artifacts[artifact_name].add_path(directory, gpu="single")

        elif artifact_name.startswith("multi-gpu"):
            artifact_name = artifact_name[len("multi-gpu") + 1 :]

            if artifact_name in _available_artifacts:
                _available_artifacts[artifact_name].multi_gpu = True
            else:
                _available_artifacts[artifact_name] = Artifact(artifact_name, multi_gpu=True)

            _available_artifacts[artifact_name].add_path(directory, gpu="multi")
        else:
            if artifact_name not in _available_artifacts:
                _available_artifacts[artifact_name] = Artifact(artifact_name)

            _available_artifacts[artifact_name].add_path(directory)



SUMMARY:
**Summary: `retrieve_available_artifacts` Function**

 retrieves and manages available artifacts, which are stored in the `_available_artifacts` dictionary.


*   **Main purpose**: This function iterates through directories on disk and populates an internal data structure with artifact information.
*   **Key implementation details**:
    *   The `_available_artifacts` dictionary is populated with `Artifact` objects, where each `Artifact` represents a directory-based asset.
    *   Directory names are used to determine the type of GPU required for the asset (single or multi).
    *   Assets are associated with paths and gpu types using the `add_path` method.
*   **Notable dependencies**:
    *   `os` module for interacting with file systems
    *   `filter` function from `itertools` for filtering directories
    *   `Dict` ( dictionary ) type hint


(Note: In a RAG system, this summary can be further used as metadata by developers to understand the functionality and implementation of this code chunk.)


key: chroma:document
string_value:
CODE:

def prepare_reports(title, header, reports, to_truncate=True):
    report = ""

    MAX_ERROR_TEXT = 3000 - len("[Truncated]")
    if not to_truncate:
        MAX_ERROR_TEXT = float("inf")

    if len(reports) > 0:
        # `text` must be less than 3001 characters in Slack SDK
        # keep some room for adding "[Truncated]" when necessary

        for idx in range(len(reports)):
            _report = header + "\n".join(reports[: idx + 1])
            new_report = f"{title}:\n```\n{_report}\n```\n"
            if len(new_report) > MAX_ERROR_TEXT:
                # `report` here has length <= 3000
                report = report + "[Truncated]"
                break
            report = new_report



SUMMARY:
**Function Summary: prepare_reports**

**Purpose:** Generate a formatted report string with truncated content if necessary.

**Primary Functionality:**
Prepares a report by combining input `title`, `header`, and `reports` into a single string, truncating the output if it exceeds 3000 characters due to "[Truncated]" addition.

**Key Implementation Details:**

* Iterates through each report in descending order using `_report = header + "\n".join(reports[: idx + 1])`.
* Checks for truncation conditions inside an `if` block (`if len(new_report) > MAX_ERROR_TEXT:`).
	+ Truncates the output with "[Truncated]" if necessary.

**Notable Dependencies/Requirements:**

* No external dependencies listed.
* Uses specific constants:
	+ `MAX_ERROR_TEXT` defined as a fixed value (3000 - 1).

Note this summary excludes unnecessary or generic details, focusing on the primary purpose and key implementation mechanics of the provided function.


key: chroma:document
string_value:
CODE:

def check_attribute_being_used(config_class, attributes, default_value, source_strings):
    """Check if any name in `attributes` is used in one of the strings in `source_strings`

    Args:
        config_class (`type`):
            The configuration class for which the arguments in its `__init__` will be checked.
        attributes (`List[str]`):
            The name of an argument (or attribute) and its variant names if any.
        default_value (`Any`):
            A default value for the attribute in `attributes` assigned in the `__init__` of `config_class`.
        source_strings (`List[str]`):
            The python source code strings in the same modeling directory where `config_class` is defined. The file
            containing the definition of `config_class` should be excluded.
    """
    attribute_used = False
    for attribute in attributes:
        for modeling_source in source_strings:
            # check if we can find `config.xxx`, `getattr(config, "xxx", ...)` or `getattr(self.config, "xxx", ...)`
            if (
                f"config.{attribute}" in modeling_source
                or f'getattr(config, "{attribute}"' in modeling_source
                or f'getattr(self.config, "{attribute}"' in modeling_source
                or (
                    "TextConfig" in config_class.__name__
                    and f"config.get_text_config().{attribute}" in modeling_source
                )
            ):
                attribute_used = True
            # Deal with multi-line cases
            elif (
                re.search(
                    rf'getattr[ \t\v\n\r\f]*\([ \t\v\n\r\f]*(self\.)?config,[ \t\v\n\r\f]*"{attribute}"',
                    modeling_source,
                )
                is not None
            ):
                attribute_used = True
            # `SequenceSummary` is called with `SequenceSummary(config)`
            elif attribute in [
                "summary_type",
                "summary_use_proj",
                "summary_activation",
                "summary_last_dropout",
                "summary_proj_to_labels",
                "summary_first_dropout",
            ]:
                if "SequenceSummary" in modeling_source:
                    attribute_used = True
            if attribute_used:
                break
        if attribute_used:
            break

    # common and important attributes, even if they do not always appear in the modeling files
    attributes_to_allow = [
        "bos_index",
        "eos_index",
        "pad_index",
        "unk_index",
        "mask_index",
        "image_token_index",  # for VLMs
        "video_token_index",
        "image_seq_length",
        "video_seq_length",
        "image_size",
        "use_cache",
        "out_features",
        "out_indices",
        "sampling_rate",
        # backbone related arguments passed to load_backbone
        "use_pretrained_backbone",
        "backbone",
        "backbone_config",
        "use_timm_backbone",
        "backbone_kwargs",
        # rope attributes may not appear directly in the modeling but are used
        "rope_theta",
        "partial_rotary_factor",
        "pretraining_tp",
    ]
    attributes_used_in_generation = ["encoder_no_repeat_ngram_size"]

    # Special cases to be allowed
    case_allowed = True
    if not attribute_used:
        case_allowed = False
        for attribute in attributes:
            # Allow if the default value in the configuration class is different from the one in `PretrainedConfig`
            if attribute in ["is_encoder_decoder"] and default_value is True:
                case_allowed = True
            elif attribute in ["tie_word_embeddings"] and default_value is False:
                case_allowed = True

            # Allow cases without checking the default value in the configuration class
            elif attribute in attributes_to_allow + attributes_used_in_generation:
                case_allowed = True
            elif attribute.endswith("_token_id"):
                case_allowed = True

            # configuration class specific cases
            if not case_allowed:
                allowed_cases = SPECIAL_CASES_TO_ALLOW.get(config_class.__name__, [])
                case_allowed = allowed_cases is True or attribute in allowed_cases



SUMMARY:
**Summary**

`check_attribute_being_used` function checks if any name in `attributes` is used in one of the strings in `source_strings`. It inspects source code for usage of attribute names, including variants like `getattr()`, and flags its presence or absence.

**Main Purpose and Functionality**
Inspect source code for attribute usage (`config.xxx`, `getattr()` constructs)

**Key Implementation Details**

* Check for direct attribute mentions (`f"config.{attribute}" in modeling_source`)
* Detect multi-line `getattr()` patterns
* Handle specific use cases like `TextConfig` and `SequenceSummary`

**Notable Dependencies or Requirements**
- `re` (regular expression library)
- `config_class` with its specific class methods (`__init__`, `get_text_config()`)


key: chroma:document
string_value:
CODE:

def check_config_attributes_being_used(config_class):
    """Check the arguments in `__init__` of `config_class` are used in the modeling files in the same directory

    Args:
        config_class (`type`):
            The configuration class for which the arguments in its `__init__` will be checked.
    """
    # Get the parameters in `__init__` of the configuration class, and the default values if any
    signature = dict(inspect.signature(config_class.__init__).parameters)
    parameter_names = [x for x in list(signature.keys()) if x not in ["self", "kwargs"]]
    parameter_defaults = [signature[param].default for param in parameter_names]

    # If `attribute_map` exists, an attribute can have different names to be used in the modeling files, and as long
    # as one variant is used, the test should pass
    reversed_attribute_map = {}
    if len(config_class.attribute_map) > 0:
        reversed_attribute_map = {v: k for k, v in config_class.attribute_map.items()}

    # Get the path to modeling source files
    config_source_file = inspect.getsourcefile(config_class)
    model_dir = os.path.dirname(config_source_file)
    # Let's check against all frameworks: as long as one framework uses an attribute, we are good.
    modeling_paths = [os.path.join(model_dir, fn) for fn in os.listdir(model_dir) if fn.startswith("modeling_")]

    # Get the source code strings
    modeling_sources = []
    for path in modeling_paths:
        if os.path.isfile(path):
            with open(path, encoding="utf8") as fp:
                modeling_sources.append(fp.read())

    unused_attributes = []
    for config_param, default_value in zip(parameter_names, parameter_defaults):
        # `attributes` here is all the variant names for `config_param`
        attributes = [config_param]
        # some configuration classes have non-empty `attribute_map`, and both names could be used in the
        # corresponding modeling files. As long as one of them appears, it is fine.
        if config_param in reversed_attribute_map:
            attributes.append(reversed_attribute_map[config_param])

        if not check_attribute_being_used(config_class, attributes, default_value, modeling_sources):
            unused_attributes.append(attributes[0])



SUMMARY:
**Summary: `check_config_attributes_being_used` Function**

This function checks if the configuration class's initialization arguments are used in the same directory's modeling files. It does this by:

1. Analyzing the init signature of the config class, identifying parameter names and default values.
2. Checking for existence and usage of attribute maps, which can map variant names to different attribute names.
3. Scanning model source files to detect attribute appearances that match the calculated attributes.

**Key Implementation Details:**

* Checks `attribute_map` for variants, allowing one usage to pass even if multiple variants exist.
* Utilizes `inspect.signature` and `os.path.join` for file path manipulation.
* Calls `check_attribute_being_used` internally to verify attribute usage against modeling sources.

**Notable Dependencies/Requirements:**

* `inspect`: For accessing signature information and getting source files.
* `os`: For handling file paths and directories.
* `config_class`, which is expected to have an `attribute_map` dictionary.

This function returns a list of unused attributes, sorted for consistency.


key: chroma:document
string_value:
CODE:

def check_config_attributes():
    """Check the arguments in `__init__` of all configuration classes are used in python files"""
    configs_with_unused_attributes = {}
    for _config_class in list(CONFIG_MAPPING.values()):
        # Skip deprecated models
        if "models.deprecated" in _config_class.__module__:
            continue
        # Some config classes are not in `CONFIG_MAPPING` (e.g. `CLIPVisionConfig`, `Blip2VisionConfig`, etc.)
        config_classes_in_module = [
            cls
            for name, cls in inspect.getmembers(
                inspect.getmodule(_config_class),
                lambda x: inspect.isclass(x)
                and issubclass(x, PretrainedConfig)
                and inspect.getmodule(x) == inspect.getmodule(_config_class),
            )
        ]
        for config_class in config_classes_in_module:
            unused_attributes = check_config_attributes_being_used(config_class)
            if len(unused_attributes) > 0:
                configs_with_unused_attributes[config_class.__name__] = unused_attributes

    if len(configs_with_unused_attributes) > 0:
        error = "The following configuration classes contain unused attributes in the corresponding modeling files:\n"
        for name, attributes in configs_with_unused_attributes.items():
            error += f"{name}: {attributes}\n"



SUMMARY:
**Function Summary:**

`check_config_attributes`: Checks if configuration class arguments in `__init__` are used in corresponding Python files and raises an error for unused attributes.

**Key Implementation Details:**

* Iterates through all configuration classes and their submodules to identify potential issue sources
* Uses `check_config_attributes_being_used` function to detect unused attributes in each configuration class

**Notable Dependencies/Requirements:**

* `CONFIG_MAPPING`: a dictionary mapping configuration classes (not defined in provided code snippet)
* `PretrainedConfig` class (assumed to be defined elsewhere)

**Primary Purpose:** To ensure that configuration class arguments are consistently used in corresponding Python files.


key: chroma:document
string_value:
CODE:

def get_jobs(workflow_run_id, token=None):
    """Extract jobs in a GitHub Actions workflow run"""

    headers = None
    if token is not None:
        headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}

    url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"
    result = requests.get(url, headers=headers).json()
    jobs = []

    try:
        jobs.extend(result["jobs"])
        pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100)

        for i in range(pages_to_iterate_over):
            result = requests.get(url + f"&page={i + 2}", headers=headers).json()
            jobs.extend(result["jobs"])

        return jobs
    except Exception:
        print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}")



SUMMARY:
**Function Summary:**

`get_jobs` extracts GitHub Actions workflow run jobs.

**Purpose:**
Extracts job data from a GitHub Actions workflow run API.

**Key Implementation Details:**

*   Uses the GitHub API to fetch job data with pagination.
*   Utilizes the `requests` library to send GET requests and parse JSON responses.

**Method Signatures:**

*   `requests.get(url, headers=headers)`: Sends a GET request to the specified URL with provided headers.
*   `.json()`: Parses the response as JSON.

**Notable Dependencies:**

*   `requests`
*   `math` (for calculating total pages)

**Interfaces Implemented:**

*   GitHub API (`https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs`)

This function uses pagination to fetch job data from the GitHub Actions workflow run API. It handles exceptions and prints errors if an unknown error occurs during execution.


key: chroma:document
string_value:
CODE:

def get_job_links(workflow_run_id, token=None):
    """Extract job names and their job links in a GitHub Actions workflow run"""

    headers = None
    if token is not None:
        headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}

    url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"
    result = requests.get(url, headers=headers).json()
    job_links = {}

    try:
        job_links.update({job["name"]: job["html_url"] for job in result["jobs"]})
        pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100)

        for i in range(pages_to_iterate_over):
            result = requests.get(url + f"&page={i + 2}", headers=headers).json()
            job_links.update({job["name"]: job["html_url"] for job in result["jobs"]})

        return job_links
    except Exception:
        print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}")



SUMMARY:
**Function Summary: get_job_links**

**Main Purpose and Functionality:**
The `get_job_links` function extracts job names and their corresponding GitHub Actions workflow run links from the Hugging Face Transformers repository.

**Key Implementation Details:**

* Uses GitHub API to fetch job data in paginated chunks (up to 100 jobs per page).
* Supports token-based authentication for request headers.
* Iterates over multiple pages of results to collect all job links.

**Notable Dependencies and Requirements:**

* Python 3.6+
* `requests` library for HTTP requests
* `math` library for calculating pagination parameters
* Hugging Face Transformers repository GitHub API permissions ( token for authentication)


key: chroma:document
string_value:
CODE:

def get_artifacts_links(worflow_run_id, token=None):
    """Get all artifact links from a workflow run"""

    headers = None
    if token is not None:
        headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}

    url = f"https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"
    result = requests.get(url, headers=headers).json()
    artifacts = {}

    try:
        artifacts.update({artifact["name"]: artifact["archive_download_url"] for artifact in result["artifacts"]})
        pages_to_iterate_over = math.ceil((result["total_count"] - 100) / 100)

        for i in range(pages_to_iterate_over):
            result = requests.get(url + f"&page={i + 2}", headers=headers).json()
            artifacts.update({artifact["name"]: artifact["archive_download_url"] for artifact in result["artifacts"]})

        return artifacts
    except Exception:
        print(f"Unknown error, could not fetch links:\n{traceback.format_exc()}")



SUMMARY:
**Summary**

This function `get_artifacts_links` retrieves URLs for artifact downloads from GitHub Actions in Hugging Face's Transformers repository. It takes a workflow run ID and an optional authentication token.

**Key Functionality:**

*   Fetches GitHub API for archive download URLs of artifacts within a workflow run.
*   Iterates over multiple pages to collect all required links until exhausted.
*   Handles pagination, total count, and error cases.

**Implementation Details:**

*   Uses `requests` library with optional authentication token via `Authorization` header.
*   Utilizes JSON parsing for receiving API response data.
*   Performs dictionary updates for efficient collection of artifact URLs.

**Notable Dependencies/Requirements:**

*   `math` for calculating pages to iterate.
*   `requests` and `json` for interacting with the GitHub API, handling responses.
*   `traceback` for error reporting in case of an issue.

**Primary Purpose:**
To fetch archive download URLs for artifacts from a specified GitHub Actions workflow run.

**Key Method Signatures:**

*   `requests.get()`: To fetch pages from the API.
*   `dict.update()`: To add new entries to the dictionary of artifact links.
*   `math.ceil()`: For calculating the number of pages to iterate over.


key: chroma:document
string_value:
CODE:

def download_artifact(artifact_name, artifact_url, output_dir, token):
    """Download a GitHub Action artifact from a URL.

    The URL is of the form `https://api.github.com/repos/huggingface/transformers/actions/artifacts/{ARTIFACT_ID}/zip`,
    but it can't be used to download directly. We need to get a redirect URL first.
    See https://docs.github.com/en/rest/actions/artifacts#download-an-artifact
    """
    headers = None
    if token is not None:
        headers = {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}

    result = requests.get(artifact_url, headers=headers, allow_redirects=False)
    download_url = result.headers["Location"]
    response = requests.get(download_url, allow_redirects=True)
    file_path = os.path.join(output_dir, f"{artifact_name}.zip")
    with open(file_path, "wb") as fp:


SUMMARY:
**Function Summary: `download_artifact`**

Downloads a GitHub Action artifact from a URL.

**Primary Functionality:**
This function downloads a zip file from a GitHub Action artifact URL. It first retrieves the redirect URL and then downloads it to a specified output directory using the provided token for authentication.

**Key Implementation Details:**

* Uses the `requests` library to send HTTP requests
* Sets headers with `Authorization` and `Accept` fields based on whether a token is provided
* Sets `allow_redirects=False` initially, but switches to `True` when downloading the redirect URL
* Retrieves the file path from the URL's `Location` header

**Notable Dependencies:**

* `requests`
* `os`

**Method Signatures and Interfaces:**
- `requests.get()`: sends a GET request to a specified URL
- `Authorization` flow for token authentication

Note that interface implementation (i.e., how it interacts with the GitHub API) is limited, but this summary should give a clear overview of its main functionality and underlying mechanisms.


key: chroma:document
string_value:
CODE:

def get_errors_from_single_artifact(artifact_zip_path, job_links=None):
    """Extract errors from a downloaded artifact (in .zip format)"""
    errors = []
    failed_tests = []
    job_name = None

    with zipfile.ZipFile(artifact_zip_path) as z:
        for filename in z.namelist():
            if not os.path.isdir(filename):
                # read the file
                if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
                    with z.open(filename) as f:
                        for line in f:
                            line = line.decode("UTF-8").strip()
                            if filename == "failures_line.txt":
                                try:
                                    # `error_line` is the place where `error` occurs
                                    error_line = line[: line.index(": ")]
                                    error = line[line.index(": ") + len(": ") :]
                                    errors.append([error_line, error])
                                except Exception:
                                    # skip un-related lines
                                    pass
                            elif filename == "summary_short.txt" and line.startswith("FAILED "):
                                # `test` is the test method that failed
                                test = line[len("FAILED ") :]
                                failed_tests.append(test)
                            elif filename == "job_name.txt":
                                job_name = line

    if len(errors) != len(failed_tests):
        raise ValueError(
            f"`errors` and `failed_tests` should have the same number of elements. Got {len(errors)} for `errors` "
            f"and {len(failed_tests)} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"
            " problem."
        )

    job_link = None
    if job_name and job_links:
        job_link = job_links.get(job_name, None)

    # A list with elements of the form (line of error, error, failed test)
    result = [x + [y] + [job_link] for x, y in zip(errors, failed_tests)]



SUMMARY:
**Code Summary: `get_errors_from_single_artifact`**

The primary purpose of this function is to extract error messages and related test information from a ZIP archive downloaded as input.

**Key Implementation Details:**

* Uses the `zipfile` module to iterate over files in the ZIP archive.
* Checks specific file contents (e.g., `failures_line.txt`, `summary_short.txt`, `job_name.txt`) for relevant data extraction.
* Processes errors and failed test information using try-except blocks.
* Raises an error if there's a mismatch between extracted error messages and failed tests.

**Method Signatures and Interfaces:**

* `get_errors_from_single_artifact(artifact_zip_path, job_links=None)`: The main function with two input parameters.
* `zip.open(filename) as f`: Used to read file contents within the ZIP archive.
* `job_links.get(job_name, None)`: Retrieves a job link based on a matching job name (if provided).

**Notable Dependencies and Requirements:**

* Python 3.x
* `zipfile` library for working with ZIP archives.
* External dependencies:
	+ `os` module for path operations.

**Primary Purpose Statement:**

Extracts errors from a downloaded artifact in XML format, mapping error lines to corresponding failed tests.


key: chroma:document
string_value:
CODE:

def get_all_errors(artifact_dir, job_links=None):
    """Extract errors from all artifact files"""

    errors = []

    paths = [os.path.join(artifact_dir, p) for p in os.listdir(artifact_dir) if p.endswith(".zip")]
    for p in paths:
        errors.extend(get_errors_from_single_artifact(p, job_links=job_links))



SUMMARY:
**Summary:**

This function retrieves all error messages from artifact files located at the specified `artifact_dir`. It searches for `.zip` files, extracts their error information using the `get_errors_from_single_artifact` function, and returns them in a list.

**Key Implementation Details:**

* Searches for `.zip` files recursively
* Calls `get_errors_from_single_artifact` on each file with optional `job_links`
* Extends errors list with results from all extracted artifacts

**Notable Dependencies:**

* `os.path.join`
* `os.listdir`
* `get_errors_from_single_artifact` ( referenced function, not shown in original snippet)

**Primary Purpose:**
Extract and return a list of error messages from artifact file zip archives.


key: chroma:document
string_value:
CODE:

def reduce_by_error(logs, error_filter=None):
    """count each error"""

    counter = Counter()
    counter.update([x[1] for x in logs])
    counts = counter.most_common()
    r = {}
    for error, count in counts:
        if error_filter is None or error not in error_filter:
            r[error] = {"count": count, "failed_tests": [(x[2], x[0]) for x in logs if x[1] == error]}

    r = dict(sorted(r.items(), key=lambda item: item[1]["count"], reverse=True))


SUMMARY:
**Summary:**

The `reduce_by_error` function clusters test log errors into a dictionary, grouped by frequency and containing failed tests. It takes in:

*   Logs with error codes as values
*   Optional error filter list
*   Counts each unique error using `Counter` and returns the results sorted by count in descending order.

**Implementation Details:**

*   Updates a `Counter` object to count unique errors from logs.
*   Creates a dictionary (`r`) to store clusters of errors, where each cluster contains:
    *   Error code
    *   Count
    *   Failed test(s) with original log index and error code
*   Sorts the final dictionary by error count in descending order.

**Dependencies/Requirements:**

*   `Counter` from collections module for counting unique errors.
*   List of expected error filter values (if provided).


key: chroma:document
string_value:
CODE:

def get_model(test):
    """Get the model name from a test method"""
    test = test.split("::")[0]
    if test.startswith("tests/models/"):
        test = test.split("/")[2]
    else:
        test = None



SUMMARY:
**Summary:**

* Function `get_model`: retrieves the model name from a test method or path.
 
**Key Implementation Details:**
 
* Splits input string by '::' and takes first part as model name
* Checks if input starts with 'tests/models/' prefix, and splits by '/' to extract model name
* Returns extracted model name, else returns None

**Notable Dependencies/Requirements:**

* None


key: chroma:document
string_value:
CODE:

def reduce_by_model(logs, error_filter=None):
    """count each error per model"""

    logs = [(x[0], x[1], get_model(x[2])) for x in logs]
    logs = [x for x in logs if x[2] is not None]
    tests = {x[2] for x in logs}

    r = {}
    for test in tests:
        counter = Counter()
        # count by errors in `test`
        counter.update([x[1] for x in logs if x[2] == test])
        counts = counter.most_common()
        error_counts = {error: count for error, count in counts if (error_filter is None or error not in error_filter)}
        n_errors = sum(error_counts.values())
        if n_errors > 0:
            r[test] = {"count": n_errors, "errors": error_counts}

    r = dict(sorted(r.items(), key=lambda item: item[1]["count"], reverse=True))


SUMMARY:
**Function Summary:** `reduce_by_model`

**Purpose:** Group and count errors by model, filtering out those excluded by the input `error_filter`.

**Implementation Details:**

* Uses list comprehension to enrich log data with the corresponding model type
* Applies error filtering using `error_filter` if provided
* Counts occurrences of each error in the `test` group using a `Counter`
* Calculates the total number of errors for each test and stores them in a dictionary

**Key Method Signatures:**

* `most_common()`: Returns the most common values and their counts via `Counter`

**Notable Dependencies/Requirements:**

* Standard library: `collections.Counter`, function `get_model`
* External dependency: None

**Context:** This code appears to be part of a logging analysis workflow, grouping errors by model type for further analysis.


key: chroma:document
string_value:
CODE:

def make_github_table(reduced_by_error):
    header = "| no. | error | status |"
    sep = "|-:|:-|:-|"
    lines = [header, sep]
    for error in reduced_by_error:
        count = reduced_by_error[error]["count"]
        line = f"| {count} | {error[:100]} |  |"
        lines.append(line)



SUMMARY:
**Function Summary: generateGitHubTable**

**Purpose:** Generate a formatted table of GitHub errors with status and count.

**Implementation Details:**

* Uses a fixed header line with four columns.
* Utilizes a separator line to split the data into rows.
* Loops through each error in `reduced_by_error`, creating a single row per error with the error name truncated to 100 characters.
* Joints all lines together using newline characters.

**Key Features:**

* `make_github_table` function takes a dictionary of errors as input (`reduced_by_error`).
* Returns a string representing the formatted table.

**Dependencies and Requirements:** 

* None explicitly defined, but assumes a Python environment with standard library access.


key: chroma:document
string_value:
CODE:

def make_github_table_per_model(reduced_by_model):
    header = "| model | no. of errors | major error | count |"
    sep = "|-:|-:|-:|-:|"
    lines = [header, sep]
    for model in reduced_by_model:
        count = reduced_by_model[model]["count"]
        error, _count = list(reduced_by_model[model]["errors"].items())[0]
        line = f"| {model} | {count} | {error[:60]} | {_count} |"
        lines.append(line)



SUMMARY:
**Summary**

**Functionality**
Create a GitHub table per model with errors, displaying:
- Model name
- No. of errors
- Major error description (truncated to 60 characters)
- Error count

**Implementation Details**
- Iterates through `reduced_by_model` dictionary
- Extracts key model data: error count and major error details
- Builds table format string with dynamic values
- Returns a formatted string with the table lines joined by newline (`\n`)

**Key Components**

* `model`: reduced_by_model dictionary keys (model names)
* `errors`: values for each model, containing error and count data
	+ error: first key-value pair item (assuming single major error per model)
	+ count: corresponding value

**Notable Dependencies/Requirements**
* None


key: chroma:document
string_value:
CODE:

def update_version_in_file(fname: str, version: str, file_type: str):
    """
    Update the version of Transformers in one file.

    Args:
        fname (`str`): The path to the file where we want to update the version.
        version (`str`): The new version to set in the file.
        file_type (`str`): The type of the file (should be a key in `REPLACE_PATTERNS`).
    """
    with open(fname, "r", encoding="utf-8", newline="\n") as f:
        code = f.read()
    re_pattern, replace = REPLACE_PATTERNS[file_type]
    replace = replace.replace("VERSION", version)
    code = re_pattern.sub(replace, code)
    with open(fname, "w", encoding="utf-8", newline="\n") as f:


SUMMARY:
**Summary:**

Updates the `version` parameter within a specified file of type `file_type`.

**Key Implementation Details:**

* Reads the contents of the file and replaces the `VERSION` placeholder with the provided `version`.
* Uses regular expression substitution (`REPLACE_PATTERNS[file_type]`) to update the version in the code.
* Writes the updated code back to the original file.

**Notable Dependencies:**

* Expects `REPLACE_PATTERNS` dictionary containing replacement patterns for different file types.


key: chroma:document
string_value:
CODE:

def update_version_in_examples(version: str):
    """
    Update the version in all examples files.

    Args:
        version (`str`): The new version to set in the examples.
    """
    for folder, directories, fnames in os.walk(PATH_TO_EXAMPLES):
        # Removing some of the folders with non-actively maintained examples from the walk
        if "research_projects" in directories:
            directories.remove("research_projects")
        if "legacy" in directories:
            directories.remove("legacy")
        for fname in fnames:
            if fname.endswith(".py"):


SUMMARY:
**Summary**

`update_version_in_examples` function updates the version of Python files in all example files within the `PATH_TO_EXAMPLES` directory. It recursively walks through the directories, filtering out inactive projects and non-Python files.

**Key Implementation Details**

* Removes outdated project folders ("research_projects" and "legacy")
* Iterates over file extensions (.py) to detect Python files
* Calls `update_version_in_file` function for each Python file to update its version with the specified `version`

**Notable Dependencies or Requirements**

* Uses `os` module for walking directories and accessing paths
* Utilizes `PATH_TO_EXAMPLES` environment variable, not defined in this code snippet.

* `update_version_in_file` function is called but not defined within this scope. It's likely a separate function that updates the version of the specified file.


key: chroma:document
string_value:
CODE:

def global_version_update(version: str, patch: bool = False):
    """
    Update the version in all needed files.

    Args:
        version (`str`): The new version to set everywhere.
        patch (`bool`, *optional*, defaults to `False`): Whether or not this is a patch release.
    """
    for pattern, fname in REPLACE_FILES.items():
        update_version_in_file(fname, version, pattern)
    if not patch:
        # We don't update the version in the examples for patch releases.


SUMMARY:
**Function Summary:**

`global_version_update`

Updates the specified version in all required files.

* Parameters:
	+ `version`: New version string
	+ `patch`: Boolean indicating patch release (default: False)

Key Implementation Details:

* Iterates through `REPLACE_FILES` dictionary to update files with pattern matches.
* Calls `update_version_in_file` method for each matched file.
* Excludes updating examples if not a patch release.

Notable Dependencies/Requirements:

* `REPLACE_FILES` dictionary
* `update_version_in_file` method
* `update_version_in_examples` function


key: chroma:document
string_value:
CODE:

def remove_conversion_scripts():
    """
    Delete the scripts that convert models from older, unsupported formats. We don't want to include these
    in release wheels because they often have to open insecure file types (pickle, Torch .bin models). This results in
    vulnerability scanners flagging us and can cause compliance issues for users with strict security policies.
    """
    model_dir = Path(PATH_TO_MODELS)
    for conversion_script in list(model_dir.glob("**/convert*.py")):


SUMMARY:
**Summary:**
`remove_conversion_scripts()` is a function that deletes Python scripts responsible for converting models from older, unsupported formats. The purpose is to prevent vulnerable file types from being included in release wheels, thereby avoiding compliance issues.

**Key Details:**

* Removes Python scripts (`convert*.py`) from the `PATH_TO_MODELS` directory using `globbing`.
* Deletes each script found using `unlink()` after listing them for removal.

**Notable Interface/Class:**

* `Path`: provides file system path manipulation.
* `glob()`: performs Unix shell-style string matching on paths.

**External Dependencies:**

None noted.


key: chroma:document
string_value:
CODE:

def get_version() -> packaging.version.Version:
    """
    Reads the current version in the main __init__.
    """
    with open(REPLACE_FILES["init"], "r") as f:
        code = f.read()
    default_version = REPLACE_PATTERNS["init"][0].search(code).groups()[0]


SUMMARY:
**Code Summary**

Retrieves the current version from the `__init__.py` file and parses it into a `packaging.version.Version` object.

**Key Details**

* Reads content from `REPLACE_FILES["init"]`
* Searches for default version in `code` using regex pattern `REPLACE_PATTERNS["init"][0]`
* Returns parsed version using `packaging.version.parse()`

**Dependencies**

* Expects `packaging` library with `version` module
* Uses regular expressions (`re`) with `search()` method


key: chroma:document
string_value:
CODE:

def pre_release_work(patch: bool = False):
    """
    Do all the necessary pre-release steps:
    - figure out the next minor release version and ask confirmation
    - update the version everywhere
    - clean-up the model list in the main README

    Args:
        patch (`bool`, *optional*, defaults to `False`): Whether or not this is a patch release.
    """
    # First let's get the default version: base version if we are in dev, bump minor otherwise.
    default_version = get_version()
    if patch and default_version.is_devrelease:
        raise ValueError("Can't create a patch version from the dev branch, checkout a released version!")
    if default_version.is_devrelease:
        default_version = default_version.base_version
    elif patch:
        default_version = f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
    else:
        default_version = f"{default_version.major}.{default_version.minor + 1}.0"

    # Now let's ask nicely if we have found the right version.
    version = input(f"Which version are you releasing? [{default_version}]")
    if len(version) == 0:
        version = default_version

    print(f"Updating version to {version}.")
    global_version_update(version, patch=patch)
    print("Deleting conversion scripts.")


SUMMARY:
**Function Summary: `pre_release_work`**

### Main Purpose and Functionality
The `preReleaseWork` function performs necessary pre-release steps for maintaining version control and cleanup. It asks confirmation, updates versions, cleans up model lists in README files, and deletes conversion scripts.

### Key Implementation Details

* Returns a default minor release version if patching, otherwise increments the minor version.
* Asks user confirmation to choose the release version.
* Updates global `version` using `global_version_update`.
* Removes conversion scripts using `remove_conversion_scripts`.

### Notable Dependencies/Requirements

* `get_version()` function
* Global variables: `base_version`, `major`, `minor`, and `micro`

### Interfaces Implemented/Inherited

* None mentioned, but may be inherited from parent class or interface.


key: chroma:document
string_value:
CODE:

def post_release_work():
    """
    Do all the necessary post-release steps:
    - figure out the next dev version and ask confirmation
    - update the version everywhere
    - clean-up the model list in the main README
    """
    # First let's get the current version
    current_version = get_version()
    dev_version = f"{current_version.major}.{current_version.minor + 1}.0.dev0"
    current_version = current_version.base_version

    # Check with the user we got that right.
    version = input(f"Which version are we developing now? [{dev_version}]")
    if len(version) == 0:
        version = dev_version

    print(f"Updating version to {version}.")


SUMMARY:
**Summary: post_release_work Function**

**Purpose and Functionality:**
The `post_release_work` function performs the necessary steps after a release, including:

* Obtaining the current version
* Confirming with the user that the next development version is correct
* Updating all versions accordingly

**Key Implementation Details:**

* Calls `get_version()` to retrieve the current version.
* Calculates the next development version by incrementing the minor version of the current version.
* Queries the user to confirm the chosen version using `input()`.
* Updates global versions using `global_version_update(version)`.

**Notable Dependencies and Requirements:**
No external dependencies mentioned. However, it is assumed that:

* `get_version()` function is available and returns a Version object with attributes such as `major`, `minor`, `base_version`.
* `global_version_update()` function is available and accepts a version string as an argument.

**Interface and Inheritance:**

* Implements no explicit interfaces. However, it assumes the existence of custom attributes for the `Version` class returned by `get_version()`.


key: chroma:document
string_value:
CODE:

def count_lines(filepath):
    """Count the number of lines in a file."""
    try:
        with open(filepath, "r") as f:
            return len(f.read().split("\n"))
    except FileNotFoundError:


SUMMARY:
**Summary:**

`count_lines` function counts the number of lines in a given file.
 
**Key Implementation Details:**
- Reads a file line by line using `open` and stores content in a variable (`f.read()`).
- Splits content into lines using newline character (`"\n"`).
- Returns the count of lines.

**Notable Dependencies:**
- [Filesystem module](https://docs.python.org/3/library/os.html#module-os) for file operations.
- This function does not use any external Python libraries, it is a fundamental file handling task.


key: chroma:document
string_value:
CODE:

def compute_parallel_nodes(line_count, max_tests_per_node=10):
    """Compute the number of parallel nodes required."""
    num_nodes = math.ceil(line_count / AVERAGE_TESTS_PER_NODES)
    if line_count < 4:
        return 1


SUMMARY:
**Summary:**

* **Main Purpose:** Computes the minimum number of parallel nodes required to execute a given line count of code with a specified maximum tests per node.

* **Key Implementation Details:**
  - Calculates `num_nodes` based on `line_count` and average tests per nodes.
  - Applies a soft cap at 1 node for small line counts (<4) to prevent division by zero errors.

* **Notable Dependencies/Requirements:**
  - `math.ceil()`: used to calculate ceiling of the division result
  - `AVERAGE_TESTS_PER_NODES`, `MAX_PARALLEL_NODES`: constants imported from parent module, representing average tests per nodes and maximum parallel nodes allowed, respectively.


key: chroma:document
string_value:
CODE:

def process_artifacts(input_file, output_file):
    # Read the JSON data from the input file
    with open(input_file, "r") as f:
        data = json.load(f)

    # Process items and build the new JSON structure
    transformed_data = {}
    for item in data.get("items", []):
        if "test_list" in item["path"]:
            key = os.path.splitext(os.path.basename(item["path"]))[0]
            transformed_data[key] = item["url"]
            parallel_key = key.split("_test")[0] + "_parallelism"
            file_path = os.path.join("test_preparation", f"{key}.txt")
            line_count = count_lines(file_path)
            transformed_data[parallel_key] = compute_parallel_nodes(line_count)

    # Remove the "generated_config" key if it exists
    if "generated_config" in transformed_data:
        del transformed_data["generated_config"]

    # Write the transformed data to the output file
    with open(output_file, "w") as f:


SUMMARY:
**Function Summary: `process_artifacts`**

### Main Purpose and Functionality

The `process_artifacts` function reads JSON data from an input file, transforms the data based on specific rules, and writes the transformed data to an output file. The primary purpose is to process artifacts related to test preparation.

**Transformation Rules:**

* If an item's path contains a "test_list" key, extract the key by removing the file extension using `os.path.basename`.
* Compute parallelism levels based on count of nodes in corresponding text files using `compute_parallel_nodes`.
* Store transformed data in a new JSON structure with `transformed_data`.

**Key Implementation Details:**

* Uses `json.load` to read input JSON file.
* Utilizes `count_lines` function to compute parallelism levels.

**Notable Dependencies or Requirements:**

* External dependency on the `os`, `json`, and `file_utils` (assuming `compute_parallel_nodes` is an external function).
* Assumed implementation of `compute_parallel_nodes` function is not shown in this code snippet.

### Additional Information:

The transformed data will be written to the output file using `json.dump`. The `"generated_config"` key from the input data will be removed if present during transformation.


key: chroma:document
string_value:
CODE:

def get_merge_commit(repo, pr_number, since_tag):
    try:
        # Use git log to find the merge commit for the PR within the given tag range
        merge_commit = next(repo.iter_commits(f"v{since_tag}...origin/main", grep=f"#{pr_number}"))
        return merge_commit
    except StopIteration:
        print(f"No merge commit found for PR #{pr_number} between tags {since_tag} and {main}")
        return None
    except GitCommandError as e:
        print(f"Error finding merge commit for PR #{pr_number}: {str(e)}")


SUMMARY:
**Summary:**

### Purpose:

This function retrieves the merge commit in a GitHub repository (using `git` commands) for a specific pull request (PR), given a tag range (`since_tag`).

### Functionality:

1. **Searches for merge commit**: Uses `repo.iter_commits` to find the first commit that matches the PR number within the specified tag range.
2. **Catches errors**:
	* `StopIteration`: No matching commit found; returns `None`.
	* `GitCommandError`: Error during command execution (e.g., invalid syntax); prints error message and returns `None`.

### Key Details:

* Method: `repo.iter_commits`
* Git log filter: `f"v{since_tag}...origin/main and grep=f#{pr_number}"`
* Returns a single merge commit object on success or `None` on failure.

**Notable Dependencies:** None

**Interfaces and Requirements:** Uses the `GitCommandError` exception class, which is part of GitHub's Git API. The repository object (`repo`) is assumed to have an `iter_commits` method.


key: chroma:document
string_value:
CODE:

def main(pr_numbers):
    repo = Repo(".")  # Initialize the Repo object for the current directory
    merge_commits = []

    tags = {}
    for tag in repo.tags:
        try:
            # Parse and sort tags, skip invalid ones
            tag_ver = version.parse(tag.name)
            tags[tag_ver] = tag
        except Exception:
            print(f"Skipping invalid version tag: {tag.name}")

    last_tag = sorted(tags)[-1]
    major_minor = f"{last_tag.major}.{last_tag.minor}.0"
    # Iterate through tag ranges to find the merge commits
    for pr in pr_numbers:
        pr = pr.split("https://github.com/huggingface/transformers/pull/")[-1]
        commit = get_merge_commit(repo, pr, major_minor)
        if commit:
            merge_commits.append(commit)

    # Sort commits by date
    merge_commits.sort(key=lambda commit: commit.committed_datetime)

    # Output the git cherry-pick commands
    print("Git cherry-pick commands to run:")
    for commit in merge_commits:


SUMMARY:
**Summary:**

### Main Functionality

The `main` function takes a list of PR numbers and generates Git cherry-pick commands to apply merge commits across tag ranges.

**Key Implementation Details:**

* **Git Repository Management**: The function initializes a Repo object for the current directory and iterates through GitHub tags to parse and sort valid version tags.
* **Tag Range Iteration**: It finds merge commits by iterating through PR numbers, splitting them into commit hashes, and using `get_merge_commit` function to retrieve them from the Git repository.

**Notable Dependencies or Requirements:**

* `Repo()` object (assumed to be a GitHub API client)
* External dependency on `version.parse()` for tag validation
* Inherited interface from `get_merge_commit()` function, which is not provided in this code snippet

### Code Summary:

```
def main(pr_numbers):
    # Initialize Repo and manage tags
    repo = Repo(".")  # ...
    tags = {}  # ...

    # Find merge commits across PR tag ranges
    for pr in pr_numbers:
        commit = get_merge_commit(repo, pr, major_minor)
        if commit:
            merge_commits.append(commit)

    # Sort commits by date and output commands
    merge_commits.sort(key=lambda commit: commit.committed_datetime)
    print("Git cherry-pick commands to run:")
    for commit in merge_commits:
        print(f"git cherry-pick {commit.hexsha} #{commit.committed_datetime}")
```

This summary aims to preserve the essence of the original code while omitting minor details and non-essential information.


