# embedding_metadata

key: chroma:document
string_value:
CODE:

class PipelineTool(Tool):
    """
    A [`Tool`] tailored towards Transformer models. On top of the class attributes of the base class [`Tool`], you will
    need to specify:

    - **model_class** (`type`) -- The class to use to load the model in this tool.
    - **default_checkpoint** (`str`) -- The default checkpoint that should be used when the user doesn't specify one.
    - **pre_processor_class** (`type`, *optional*, defaults to [`AutoProcessor`]) -- The class to use to load the
      pre-processor
    - **post_processor_class** (`type`, *optional*, defaults to [`AutoProcessor`]) -- The class to use to load the
      post-processor (when different from the pre-processor).

    Args:
        model (`str` or [`PreTrainedModel`], *optional*):
            The name of the checkpoint to use for the model, or the instantiated model. If unset, will default to the
            value of the class attribute `default_checkpoint`.
        pre_processor (`str` or `Any`, *optional*):
            The name of the checkpoint to use for the pre-processor, or the instantiated pre-processor (can be a
            tokenizer, an image processor, a feature extractor or a processor). Will default to the value of `model` if
            unset.
        post_processor (`str` or `Any`, *optional*):
            The name of the checkpoint to use for the post-processor, or the instantiated pre-processor (can be a
            tokenizer, an image processor, a feature extractor or a processor). Will default to the `pre_processor` if
            unset.
        device (`int`, `str` or `torch.device`, *optional*):
            The device on which to execute the model. Will default to any accelerator available (GPU, MPS etc...), the
            CPU otherwise.
        device_map (`str` or `dict`, *optional*):
            If passed along, will be used to instantiate the model.
        model_kwargs (`dict`, *optional*):
            Any keyword argument to send to the model instantiation.
        token (`str`, *optional*):
            The token to use as HTTP bearer authorization for remote files. If unset, will use the token generated when
            running `huggingface-cli login` (stored in `~/.huggingface`).
        hub_kwargs (additional keyword arguments, *optional*):
            Any additional keyword argument to send to the methods that will load the data from the Hub.
    """

    pre_processor_class = AutoProcessor
    model_class = None
    post_processor_class = AutoProcessor
    default_checkpoint = None
    description = "This is a pipeline tool"
    name = "pipeline"
    inputs = {"prompt": str}
    output_type = str

    def __init__(
        self,
        model=None,
        pre_processor=None,
        post_processor=None,
        device=None,
        device_map=None,
        model_kwargs=None,
        token=None,
        **hub_kwargs,
    ):
        if not is_torch_available():
            raise ImportError("Please install torch in order to use this tool.")

        if not is_accelerate_available():
            raise ImportError("Please install accelerate in order to use this tool.")

        if model is None:
            if self.default_checkpoint is None:
                raise ValueError("This tool does not implement a default checkpoint, you need to pass one.")
            model = self.default_checkpoint
        if pre_processor is None:
            pre_processor = model

        self.model = model
        self.pre_processor = pre_processor
        self.post_processor = post_processor
        self.device = device
        self.device_map = device_map
        self.model_kwargs = {} if model_kwargs is None else model_kwargs
        if device_map is not None:
            self.model_kwargs["device_map"] = device_map
        self.hub_kwargs = hub_kwargs
        self.hub_kwargs["token"] = token

        super().__init__()

    def setup(self):
        """
        Instantiates the `pre_processor`, `model` and `post_processor` if necessary.
        """
        if isinstance(self.pre_processor, str):
            self.pre_processor = self.pre_processor_class.from_pretrained(self.pre_processor, **self.hub_kwargs)

        if isinstance(self.model, str):
            self.model = self.model_class.from_pretrained(self.model, **self.model_kwargs, **self.hub_kwargs)

        if self.post_processor is None:
            self.post_processor = self.pre_processor
        elif isinstance(self.post_processor, str):
            self.post_processor = self.post_processor_class.from_pretrained(self.post_processor, **self.hub_kwargs)

        if self.device is None:
            if self.device_map is not None:
                self.device = list(self.model.hf_device_map.values())[0]
            else:
                self.device = PartialState().default_device

        if self.device_map is None:
            self.model.to(self.device)

        super().setup()

    def encode(self, raw_inputs):
        """
        Uses the `pre_processor` to prepare the inputs for the `model`.
        """
        return self.pre_processor(raw_inputs)

    def forward(self, inputs):
        """
        Sends the inputs through the `model`.
        """
        with torch.no_grad():
            return self.model(**inputs)

    def decode(self, outputs):
        """
        Uses the `post_processor` to decode the model output.
        """
        return self.post_processor(outputs)

    def __call__(self, *args, **kwargs):
        args, kwargs = handle_agent_inputs(*args, **kwargs)

        if not self.is_initialized:
            self.setup()

        encoded_inputs = self.encode(*args, **kwargs)

        tensor_inputs = {k: v for k, v in encoded_inputs.items() if isinstance(v, torch.Tensor)}
        non_tensor_inputs = {k: v for k, v in encoded_inputs.items() if not isinstance(v, torch.Tensor)}

        encoded_inputs = send_to_device(tensor_inputs, self.device)
        outputs = self.forward({**encoded_inputs, **non_tensor_inputs})
        outputs = send_to_device(outputs, "cpu")
        decoded_outputs = self.decode(outputs)



SUMMARY:
**PipelineTool Class Summary**

The `PipelineTool` class is a custom tool for Transformer models, built on top of the `Tool` base class. It serves as a pipeline for model loading, preprocessing, and postprocessing.

**Key Functionality:**

1. Initialization: `setup()` method instantiates and configures the pre-processor, model, and post-processor.
2. Encoding: `encode(raw_inputs)` applies the pre-processor to prepare inputs for the model.
3. Forward Pass: `forward(inputs)` sends inputs through the model and returns outputs.
4. Decoding: `decode(outputs)` applies the post-processor to decode model outputs.

**Implementation Details:**

* Uses `AutoProcessor` and `PreTrainedModel` classes from Hugging Face Hub.
* Custom device handling and token authorization for remote files.
* Supports optional keyword arguments for model instantiation (`model_kwargs`) and hub-related parameters (`hub_kwargs`).

**Notable Dependencies/Requirements:**

* `torch`
* `accelerate` (for acceleration support on GPUs/MPS)
* Hugging Face Hub for pre-trained models and processors

**Primary Class Attributes:**

* `pre_processor_class`: defaults to `AutoProcessor`.
* `model_class`: optional, set by user or defaults to `None`.
* `post_processor_class`: defaults to `AutoProcessor` (optional).
* `default_checkpoint`: default model checkpoint.

This class is designed for easy use and flexibility in working with Transformer models in a pipeline-like fashion.


key: chroma:document
string_value:
CODE:

def launch_gradio_demo(tool_class: Tool):
    """
    Launches a gradio demo for a tool. The corresponding tool class needs to properly implement the class attributes
    `inputs` and `output_type`.

    Args:
        tool_class (`type`): The class of the tool for which to launch the demo.
    """
    try:
        import gradio as gr
    except ImportError:
        raise ImportError("Gradio should be installed in order to launch a gradio demo.")

    tool = tool_class()

    def fn(*args, **kwargs):
        return tool(*args, **kwargs)

    TYPE_TO_COMPONENT_CLASS_MAPPING = {
        "image": gr.Image,
        "audio": gr.Audio,
        "string": gr.Textbox,
        "integer": gr.Textbox,
        "number": gr.Textbox,
    }

    gradio_inputs = []
    for input_name, input_details in tool_class.inputs.items():
        input_gradio_component_class = TYPE_TO_COMPONENT_CLASS_MAPPING[input_details["type"]]
        new_component = input_gradio_component_class(label=input_name)
        gradio_inputs.append(new_component)

    output_gradio_componentclass = TYPE_TO_COMPONENT_CLASS_MAPPING[tool_class.output_type]
    gradio_output = output_gradio_componentclass(label=input_name)

    gr.Interface(
        fn=fn,
        inputs=gradio_inputs,
        outputs=gradio_output,
        title=tool_class.__name__,
        article=tool.description,


SUMMARY:
**Summary:**

Launches a Gradio demo for a tool class. 

* Launches Gradio with the provided `tool_class`.
* Dynamically creates input and output components based on the tool's attributes (`inputs` and `output_type`).
* Uses an interface-based design to define the UI flow.
	+ Inputs are mapped from the `inputs` attribute using a pre-defined mapping of types to component classes.
	+ Output is created based on the `output_type`.
* Raises an error if Gradio is not installed.

**Key Implementation Details:**

* Dynamically creates input and output components for the tool class.
* Uses a lookup table (`TYPE_TO_COMPONENT_CLASS_MAPPING`) to map input types to component classes.

**Notable Dependencies or Requirements:**

* `gradio`: installed or imported with an error handling mechanism.


key: chroma:document
string_value:
CODE:

def load_tool(task_or_repo_id, model_repo_id=None, token=None, **kwargs):
    """
    Main function to quickly load a tool, be it on the Hub or in the Transformers library.

    <Tip warning={true}>

    Loading a tool means that you'll download the tool and execute it locally.
    ALWAYS inspect the tool you're downloading before loading it within your runtime, as you would do when
    installing a package using pip/npm/apt.

    </Tip>

    Args:
        task_or_repo_id (`str`):
            The task for which to load the tool or a repo ID of a tool on the Hub. Tasks implemented in Transformers
            are:

            - `"document_question_answering"`
            - `"image_question_answering"`
            - `"speech_to_text"`
            - `"text_to_speech"`
            - `"translation"`

        model_repo_id (`str`, *optional*):
            Use this argument to use a different model than the default one for the tool you selected.
        token (`str`, *optional*):
            The token to identify you on hf.co. If unset, will use the token generated when running `huggingface-cli
            login` (stored in `~/.huggingface`).
        kwargs (additional keyword arguments, *optional*):
            Additional keyword arguments that will be split in two: all arguments relevant to the Hub (such as
            `cache_dir`, `revision`, `subfolder`) will be used when downloading the files for your tool, and the others
            will be passed along to its init.
    """
    if task_or_repo_id in TOOL_MAPPING:
        tool_class_name = TOOL_MAPPING[task_or_repo_id]
        main_module = importlib.import_module("transformers")
        tools_module = main_module.agents
        tool_class = getattr(tools_module, tool_class_name)
        return tool_class(model_repo_id, token=token, **kwargs)
    else:
        logger.warning_once(
            f"You're loading a tool from the Hub from {model_repo_id}. Please make sure this is a source that you "
            f"trust as the code within that tool will be executed on your machine. Always verify the code of "
            f"the tools that you load. We recommend specifying a `revision` to ensure you're loading the "
            f"code that you have checked."
        )


SUMMARY:
**Summary:**

The `load_tool` function loads a tool from either the Hub or the Transformers library. It takes in five parameters:

*   `task_or_repo_id`: a string representing the task or repository ID of the tool to load
*   `model_repo_id`: an optional string argument for using a different model than the default one
*   `token`: an optional string argument for identifying itself on hf.co
*   `kwargs`: additional keyword arguments

The function first checks if the provided `task_orRepo_id` exists in a mapping called `TOOL_MAPPING`. If it does, it loads the corresponding tool class and instantiates it with the given parameters. Otherwise, it logs a warning and returns a `Tool` object loaded from the Hub.

**Key Implementation Details:**

*   The function uses an importlib import statement to dynamically load modules from the Transformers library.
*   It utilizes a dictionary called `TOOL_MAPPING` to map repository IDs to their corresponding tool class names.
*   The function logs a warning message using a logger object when loading a tool from the Hub.

**Notable Dependencies and Requirements:**

*   The function imports modules from the `transformers` library
*   It relies on the `importlib` module for dynamic imports
*   It uses a logging system to issue warnings


key: chroma:document
string_value:
CODE:

def add_description(description):
    """
    A decorator that adds a description to a function.
    """

    def inner(func):
        func.description = description
        func.name = func.__name__
        return func



SUMMARY:
**Summary**

`add_description` is a decorator function that sets the `description` and `name` attributes of an object (function) passed to it.

**Detailed Analysis**

* **Main Purpose and Functionality**: The main purpose of this decorator is to add metadata (description) to a function.
* **Implementation Details**:
	+ The decorated function retains its original `__name__`.
	+ A new attribute `description` is added to the function object with the provided description value.
	+ The inner function returns the modified function object with the added attributes.
* **Notable Dependencies and Requirements**: None.
 
**Summary Output**
```
add_description: Decorator that adds a description to a function.

  Sets description and name attributes on a function object.

[ inner function ]
  func.description = description
  func.name = func.__name__
```


key: chroma:document
string_value:
CODE:
## Will move to the Hub
class EndpointClient:
    def __init__(self, endpoint_url: str, token: Optional[str] = None):
        self.headers = {
            **build_hf_headers(token=token),
            "Content-Type": "application/json",
        }
        self.endpoint_url = endpoint_url

    @staticmethod
    def encode_image(image):
        _bytes = io.BytesIO()
        image.save(_bytes, format="PNG")
        b64 = base64.b64encode(_bytes.getvalue())
        return b64.decode("utf-8")

    @staticmethod
    def decode_image(raw_image):
        if not is_vision_available():
            raise ImportError(
                "This tool returned an image but Pillow is not installed. Please install it (`pip install Pillow`)."
            )

        from PIL import Image

        b64 = base64.b64decode(raw_image)
        _bytes = io.BytesIO(b64)
        return Image.open(_bytes)

    def __call__(
        self,
        inputs: Optional[Union[str, Dict, List[str], List[List[str]]]] = None,
        params: Optional[Dict] = None,
        data: Optional[bytes] = None,
        output_image: bool = False,
    ) -> Any:
        # Build payload
        payload = {}
        if inputs:
            payload["inputs"] = inputs
        if params:
            payload["parameters"] = params

        # Make API call
        response = get_session().post(self.endpoint_url, headers=self.headers, json=payload, data=data)

        # By default, parse the response for the user.
        if output_image:
            return self.decode_image(response.content)
        else:


SUMMARY:
**EndpointClient Class Summary**

### Main Purpose and Functionality
`EndpointClient` is a class used to interact with an external API endpoint, handling requests and responses.

```python
class EndpointClient:
    def __init__(self, endpoint_url: str, token: Optional[str] = None):
        ...
```

### Key Implementation Details

*   The `__call__()` method constructs the API payload and makes a POST request to the endpoint.
*   Optional parameters can be passed in to customize the request, including input data (`inputs`), query parameters (`params`), and output image flag.

```python
def __call__(
    self,
    inputs: Optional[Union[str, Dict, List[str], List[List[str]]]] = None,
    params:Optional[Dict] = None,
    data:data = None,
    output_image: bool = False,
)-> Any:
    ...
```

### Notable Dependencies or Requirements

*   It relies on the `get_session()` function, which is not defined in this snippet but seems to provide an HTTP client session.
*   The implementation requires the `Pillow` library for image decoding (`from PIL import Image`) when requested.

```python
import io
from PIL import Image
...

@staticmethod
def decode_image(raw_image):
    ...
    
from PIL import Image
...
```

This summary provides a concise and focused overview of the `EndpointClient` class, highlighting its key functionality, dependencies, and notable requirements.


key: chroma:document
string_value:
CODE:

class ToolCollection:
    """
    Tool collections enable loading all Spaces from a collection in order to be added to the agent's toolbox.

    > [!NOTE]
    > Only Spaces will be fetched, so you can feel free to add models and datasets to your collection if you'd
    > like for this collection to showcase them.

    Args:
        collection_slug (str):
            The collection slug referencing the collection.
        token (str, *optional*):
            The authentication token if the collection is private.

    Example:

    ```py
    >>> from transformers import ToolCollection, ReactCodeAgent

    >>> image_tool_collection = ToolCollection(collection_slug="huggingface-tools/diffusion-tools-6630bb19a942c2306a2cdb6f")
    >>> agent = ReactCodeAgent(tools=[*image_tool_collection.tools], add_base_tools=True)

    >>> agent.run("Please draw me a picture of rivers and lakes.")
    ```
    """

    def __init__(self, collection_slug: str, token: Optional[str] = None):
        self._collection = get_collection(collection_slug, token=token)
        self._hub_repo_ids = {item.item_id for item in self._collection.items if item.item_type == "space"}


SUMMARY:
**Class Summary: ToolCollection**

Purpose:
The `ToolCollection` class enables loading all Spaces from a collection to be added to an agent's toolbox.

Main Functionality:

* Initializes the collection with a slug and optional token for authentication
* Fetched Spaces are extracted and stored in `_hub_repo_ids`
* Tools from hub repos identified by `_hub_repo_ids` are instantiated

Key Details:

* Uses `get_collection` function to fetch Space data
* _collection: The fetched Space data, where item_type is always "space"
* tools: A set of Tool instances created from hub repos in `self._hub_repo_ids`

Notable Dependencies:

* get_collection()
* Tool.from_hub(repo_id)

**Example Usage:**

```python
image_tool_collection = ToolCollection(collection_slug="huggingface-tools/diffusion-tools-6630bb19a942c2306a2cdb6f")
agent = ReactCodeAgent(tools=[*image_tool_collection.tools], add_base_tools=True)
```

This summary captures the essence of the `ToolCollection` class, highlighting its purpose, key implementation details, and notable dependencies.


key: chroma:document
string_value:
CODE:

def tool(tool_function: Callable) -> Tool:
    """
    Converts a function into an instance of a Tool subclass.

    Args:
        tool_function: Your function. Should have type hints for each input and a type hint for the output.
        Should also have a docstring description including an 'Args:' part where each argument is described.
    """
    parameters = get_json_schema(tool_function)["function"]
    if "return" not in parameters:
        raise TypeHintParsingException("Tool return type not found: make sure your function has a return type hint!")
    class_name = f"{parameters['name'].capitalize()}Tool"

    class SpecificTool(Tool):
        name = parameters["name"]
        description = parameters["description"]
        inputs = parameters["parameters"]["properties"]
        output_type = parameters["return"]["type"]

        @wraps(tool_function)
        def forward(self, *args, **kwargs):
            return tool_function(*args, **kwargs)

    original_signature = inspect.signature(tool_function)
    new_parameters = [inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD)] + list(
        original_signature.parameters.values()
    )
    new_signature = original_signature.replace(parameters=new_parameters)
    SpecificTool.forward.__signature__ = new_signature

    SpecificTool.__name__ = class_name


SUMMARY:
**Summary: Function Conversion to Tool Instance**

This function converts a given Python function into an instance of a `SpecificTool` subclass. The key functionality involves:

* Extracting type hints and schema from the original function using JSON.
* Creating a class with attributes matching the function's name, description, and input/output types.
* Defining a method (`forward`) that wraps the original function with self reference.
* Updating the class signature to match the new method signature.

**Key Details:**

* Uses `tool_function` type hints for inputs and output types.
* Involves JSON schema extraction using `get_json_schema`.
* Wraps the original function in a `self`-referenced method using `@wraps`.

**Dependencies/Requirements:**

* `inspect` module for signature manipulation.
* `get_json_schema` function (not shown).
* `Tool` and `SpecificTool` subclass definitions (not shown).


key: chroma:document
string_value:
CODE:

class InterpreterError(ValueError):
    """
    An error raised when the interpretor cannot evaluate a Python expression, due to syntax error or unsupported
    operations.
    """



SUMMARY:
**Summary:**

The `InterpreterError` class is a custom exception that inherits from `ValueError`. Its primary purpose is to represent errors encountered during the evaluation of Python expressions. The class provides no additional implementation; it serves as a mere placeholder for potential future code.

**Implementation Details:**

* Raises an error due to syntax or unsupported operations.
* Inherited from `ValueError`.
* No additional implementation provided.

**Notable Dependencies/Requirements:**

* None explicitly stated.
* inherits from `ValueError` (base class).


key: chroma:document
string_value:
CODE:

class BreakException(Exception):


SUMMARY:
**Summary: Custom Exception Class**

The `BreakException` class is a custom exception class that inherits from the built-in `Exception` class. It serves as a custom error type for breaking out of a loop or terminating an abnormal program flow.

**Key Implementation Details:**
- No additional logic is added to override default behavior.
 

**Notable Dependencies/Requirements:** 

- Built-in `Exception` class (Python Standard Library)


key: chroma:document
string_value:
CODE:

class ContinueException(Exception):


SUMMARY:
**Class Summary:**

`ContinueException`

*   **Main Purpose and Functionality:** Custom exception class to signal that the execution should continue.

*   **Key Implementation Details:** None, as it's a minimalist exception class with no additional functionality or custom behavior.

*   **Notable Dependencies or Requirements:** None.

This summary preserves the exact class name, `ContinueException`, and accurately reflects its intended purpose. The key implementation detail is explicitly stated as none, but if there were any notable methods or attributes in this class, they would be included here based on the original code snippet, which is currently empty. However, since it's a simple exception class with no additional functionality, this summary provides the most accurate and concise representation possible without adding unnecessary information.


key: chroma:document
string_value:
CODE:

class ReturnException(Exception):
    def __init__(self, value):


SUMMARY:
**Summary: Custom Exception Class**

* **Purpose**: Defines a custom exception class `ReturnException` to handle return value errors.
* **Functionality**: Initializes an instance with an error value and stores it in the `value` attribute.
* **Key Implementation Detail**: The `__init__` method directly assigns the provided value to the instance's `value` attribute.

**Notable Details:**

* Throws the exception when an error occurs during return.
* Interface: Extends from the Python built-in `Exception` class.


key: chroma:document
string_value:
CODE:

def get_iterable(obj):
    if isinstance(obj, list):
        return obj
    elif hasattr(obj, "__iter__"):
        return list(obj)
    else:


SUMMARY:
**Function Summary**

`get_iterable(func)` converts an object into an iterable sequence.

**Implementation Details**

* Checks if the input `obj` is a list and returns it directly.
* Uses `hasattr` to check for the presence of the `__iter__` attribute, indicating potential iteration capability. If present, wraps it in a list using `list(obj)`.
* Raises an `InterpreterError` if the object is neither iterable nor lists.

**Notable Dependencies/Requirements**

* None explicitly listed
* Assumes access to Python's built-in `list`, `object`, and `TypeError` classes

Note: It appears that there might be a small typo in the function signature. The last line should likely be an `IterableError` instead of `InterpreterError` or `TypeError`. A corrected version would suggest:

    def get_iterable(obj):
        if isinstance(obj, list):
            return obj
        elif hasattr(obj, "__iter__"):
            return list(obj)
        else:
            raise IterableError("Object is not iterable")


key: chroma:document
string_value:
CODE:

def evaluate_unaryop(expression, state, static_tools, custom_tools):
    operand = evaluate_ast(expression.operand, state, static_tools, custom_tools)
    if isinstance(expression.op, ast.USub):
        return -operand
    elif isinstance(expression.op, ast.UAdd):
        return operand
    elif isinstance(expression.op, ast.Not):
        return not operand
    elif isinstance(expression.op, ast.Invert):
        return ~operand
    else:


SUMMARY:
**Summary: Unary Operator Evaluation Function**

Evaluates unary operators in an abstract syntax tree (AST) expression.

**Purpose and Functionality:**
The `evaluate_unaryop` function takes an AST expression, state, static tools, and custom tools as input. It evaluates the unary operation represented by the `operand` and applies the corresponding operator:
- `-` for `ast.USub`
- `+` for `ast.UAdd`
- `not` for `ast.Not`
- `~` for `ast.Invert`

**Key Implementation Details:**

* The function uses `evaluate_ast` to evaluate the operand.
* It supports four unary operators and raises an error for unsupported operations.

**Notable Dependencies and Requirements:**

* `static_tools` and `custom_tools`: pass custom tools as arguments (no specific details provided)
* `ast.USub`, `ast.UAdd`, `ast.Not`, and `ast.Invert` classes are used from the Python standard library `ast`
* `InterpreterError` class is assumed to be defined elsewhere in the codebase


key: chroma:document
string_value:
CODE:

def evaluate_lambda(lambda_expression, state, static_tools, custom_tools):
    args = [arg.arg for arg in lambda_expression.args.args]

    def lambda_func(*values):
        new_state = state.copy()
        for arg, value in zip(args, values):
            new_state[arg] = value
        return evaluate_ast(lambda_expression.body, new_state, static_tools, custom_tools)



SUMMARY:
**Function Summary**

`evaluate_lambda`: Evaluates a Lambda expression function.

**Primary Purpose**: Creates and evaluates a Lambda function with the given input parameters.

**Key Implementation Details**:

* Uses `zip` to iterate over argument values.
* Assigns new value to state using dictionary updates (`new_state[arg] = value`).
* Calls `evaluate_ast` to evaluate the Body of the Lambda Expression.

**Notable Dependencies and Requirements**:

* `argparse` (accessed via `lambda_expression.args`)
* `statertools` and `custom_tools` classes
* `evaluate_ast` function

**Method Signature**:
```
def lambda_func(*values) -> <result type>
```


key: chroma:document
string_value:
CODE:

def evaluate_while(while_loop, state, static_tools, custom_tools):
    max_iterations = 1000
    iterations = 0
    while evaluate_ast(while_loop.test, state, static_tools, custom_tools):
        for node in while_loop.body:
            try:
                evaluate_ast(node, state, static_tools, custom_tools)
            except BreakException:
                return None
            except ContinueException:
                break
        iterations += 1
        if iterations > max_iterations:
            raise InterpreterError(f"Maximum number of {max_iterations} iterations in While loop exceeded")


SUMMARY:
**Concise Summary:**

`evaluate_while` function evaluates a while loop with the following key characteristics:

* Primary purpose: Execute the condition and body of a while loop, raising an error if maximum iterations are exceeded.
* Key implementation details:
	+ Evaluate `test` condition using `evaluate_ast`.
	+ Iterate over the `body` nodes, executing and catching `BreakException` and `ContinueException`.
	+ Increment iteration counter; raise InterpreterError upon exceeding `max_iterations`.
* Notable dependencies and requirements:
	+ `static_tools`, `custom_tools`, and `InterpreterError` (not shown in this code snippet).
	+ Inherited/implemented interfaces: `evaluate_ast`, `InterpreterError`.


key: chroma:document
string_value:
CODE:

def create_function(func_def, state, static_tools, custom_tools):
    def new_func(*args, **kwargs):
        func_state = state.copy()
        arg_names = [arg.arg for arg in func_def.args.args]
        default_values = [evaluate_ast(d, state, static_tools, custom_tools) for d in func_def.args.defaults]

        # Apply default values
        defaults = dict(zip(arg_names[-len(default_values) :], default_values))

        # Set positional arguments
        for name, value in zip(arg_names, args):
            func_state[name] = value

        # # Set keyword arguments
        for name, value in kwargs.items():
            func_state[name] = value

        # Handle variable arguments
        if func_def.args.vararg:
            vararg_name = func_def.args.vararg.arg
            func_state[vararg_name] = args

        if func_def.args.kwarg:
            kwarg_name = func_def.args.kwarg.arg
            func_state[kwarg_name] = kwargs

        # Set default values for arguments that were not provided
        for name, value in defaults.items():
            if name not in func_state:
                func_state[name] = value

        # Update function state with self and __class__
        if func_def.args.args and func_def.args.args[0].arg == "self":
            if args:
                func_state["self"] = args[0]
                func_state["__class__"] = args[0].__class__

        result = None
        try:
            for stmt in func_def.body:
                result = evaluate_ast(stmt, func_state, static_tools, custom_tools)
        except ReturnException as e:
            result = e.value
        return result



SUMMARY:
**Function Summary: `create_function`**

The `create_function` function creates a new function by combining the provided input parameters. The primary purpose is to execute the function body while binding arguments and maintaining state.

**Key Implementation Details:**

*   Evaluates default argument values using `evaluate_ast`.
*   Sets positional, keyword, and variable arguments.
*   Updates the function state with `self` and `__class__`.
*   Executes the function body using a loop and retrieves the result.

**Notable Dependencies or Requirements:**

*   The `state` parameter is assumed to be an instance of `OrderedDict`.
*   The `static_tools` and `custom_tools` parameters are used for evaluating AST nodes.
*   The `evaluate_ast` function is not shown in the provided code snippet but is likely a reusable utility.

**Interfaces/Classes:**

*   `func_def`: An attribute dictionary representing the target function definition (`func_def.args`, `func_def.body`).
*   `OrderDict` and `ReturnException`: External dependencies (assumed to be part of a larger AST evaluation framework).


key: chroma:document
string_value:
CODE:

def create_class(class_name, class_bases, class_body):
    class_dict = {}
    for key, value in class_body.items():
        class_dict[key] = value


SUMMARY:
**Summary:**

`create_class` function creates a new class dynamically using the `type()` constructor.

* Purpose: Creates a new class with a specified name and inherits from given base classes.
* Key Implementation Details:
	+ Applies class body to create an instance dictionary (`class_dict`) mapping method names to functions.
	+ Returns a dynamically created type with the provided name, base classes, and instance dictionary.
* Dependencies/Requirements:
	None
* Relevant Classes/Functions:
	`type()`

**Code Chunk:**

```python
def create_class(class_name, class_bases, class_body):
    class_dict = {}
    for key, value in class_body.items():
        class_dict[key] = value
    return type(class_name, tuple(class_bases), class_dict)
```

Note: The `create_class` function uses the built-in Python `type()` function to create a new class. This is equivalent to defining the class manually using the following syntax:

```python
class ClassName:
    def __init__(self):
        pass

    # add methods and attributes here
```

This dynamic approach allows for more flexibility in creating classes at runtime.


key: chroma:document
string_value:
CODE:

def evaluate_function_def(func_def, state, static_tools, custom_tools):
    custom_tools[func_def.name] = create_function(func_def, state, static_tools, custom_tools)


SUMMARY:
**Summary:**
Evaluates and registers a function definition in the `custom_tools` dictionary.

**Key Implementation Details:**

* Registers a created function with the specified name using `create_function`.
* Returns the registered function from the `custom_tools` dictionary.

**Notable Dependencies/Requirements:**

* `state`
* `static_tools`
* `custom_tools`

Note that this summary focuses on the crucial aspects of the code chunk, eliminating unnecessary information. The technical terms are used precisely to ensure clarity and accuracy in the RAG system.


key: chroma:document
string_value:
CODE:

def evaluate_class_def(class_def, state, static_tools, custom_tools):
    class_name = class_def.name
    bases = [evaluate_ast(base, state, static_tools, custom_tools) for base in class_def.bases]
    class_dict = {}

    for stmt in class_def.body:
        if isinstance(stmt, ast.FunctionDef):
            class_dict[stmt.name] = evaluate_function_def(stmt, state, static_tools, custom_tools)
        elif isinstance(stmt, ast.Assign):
            for target in stmt.targets:
                if isinstance(target, ast.Name):
                    class_dict[target.id] = evaluate_ast(stmt.value, state, static_tools, custom_tools)
                elif isinstance(target, ast.Attribute):
                    class_dict[target.attr] = evaluate_ast(stmt.value, state, static_tools, custom_tools)
        else:
            raise InterpreterError(f"Unsupported statement in class body: {stmt.__class__.__name__}")

    new_class = type(class_name, tuple(bases), class_dict)
    state[class_name] = new_class


SUMMARY:
**Function Summary**

`evaluate_class_def`

Evaluates and creates a new Python class definition based on the provided abstract syntax tree (AST) node.

**Main Purpose**
Creates a new Python class by recursively evaluating its attributes, methods, and base classes.

**Key Implementation Details**

- Evaluates base classes using `evaluate_ast`.
- Iterates over method function definitions and assigns results to the class dictionary.
- Handles assignment statements by looking up variable assignments in the class dictionary.
- Creates a new class definition using the `type` constructor with the class name, base classes, and attribute table.

**Notable Dependencies and Requirements**

- Utilizes `state`, `static_tools`, and `custom_tools` dictionaries for intermediate results.
- Imports Python AST (Abstract Syntax Tree) library (`ast`).
- Expects a valid Python abstract syntax tree node.


key: chroma:document
string_value:
CODE:

def evaluate_augassign(expression, state, static_tools, custom_tools):
    # Helper function to get current value and set new value based on the target type
    def get_current_value(target):
        if isinstance(target, ast.Name):
            return state.get(target.id, 0)
        elif isinstance(target, ast.Subscript):
            obj = evaluate_ast(target.value, state, static_tools, custom_tools)
            key = evaluate_ast(target.slice, state, static_tools, custom_tools)
            return obj[key]
        elif isinstance(target, ast.Attribute):
            obj = evaluate_ast(target.value, state, static_tools, custom_tools)
            return getattr(obj, target.attr)
        elif isinstance(target, ast.Tuple):
            return tuple(get_current_value(elt) for elt in target.elts)
        elif isinstance(target, ast.List):
            return [get_current_value(elt) for elt in target.elts]
        else:
            raise InterpreterError("AugAssign not supported for {type(target)} targets.")

    current_value = get_current_value(expression.target)
    value_to_add = evaluate_ast(expression.value, state, static_tools, custom_tools)

    # Determine the operation and apply it
    if isinstance(expression.op, ast.Add):
        if isinstance(current_value, list):
            if not isinstance(value_to_add, list):
                raise InterpreterError(f"Cannot add non-list value {value_to_add} to a list.")
            updated_value = current_value + value_to_add
        else:
            updated_value = current_value + value_to_add
    elif isinstance(expression.op, ast.Sub):
        updated_value = current_value - value_to_add
    elif isinstance(expression.op, ast.Mult):
        updated_value = current_value * value_to_add
    elif isinstance(expression.op, ast.Div):
        updated_value = current_value / value_to_add
    elif isinstance(expression.op, ast.Mod):
        updated_value = current_value % value_to_add
    elif isinstance(expression.op, ast.Pow):
        updated_value = current_value**value_to_add
    elif isinstance(expression.op, ast.FloorDiv):
        updated_value = current_value // value_to_add
    elif isinstance(expression.op, ast.BitAnd):
        updated_value = current_value & value_to_add
    elif isinstance(expression.op, ast.BitOr):
        updated_value = current_value | value_to_add
    elif isinstance(expression.op, ast.BitXor):
        updated_value = current_value ^ value_to_add
    elif isinstance(expression.op, ast.LShift):
        updated_value = current_value << value_to_add
    elif isinstance(expression.op, ast.RShift):
        updated_value = current_value >> value_to_add
    else:
        raise InterpreterError(f"Operation {type(expression.op).__name__} is not supported.")

    # Update the state
    set_value(expression.target, updated_value, state, static_tools, custom_tools)



SUMMARY:
**Function Summary:**

`evaluate_augassign(expression, state, static_tools, custom_tools)`

Evaluates an augmented assignment expression and updates the target value based on the operation and addition/subtraction values.

**Functionality:**

1. Evaluates the target value using `get_current_value` to retrieve its current state.
2. Applies the specified operation (e.g., addition, subtraction, multiplication, division) between the target value (`current_value`) and the addition/subtraction value (`value_to_add`).
3. Updates the target value with the result of the operation.

**Key Implementation Details:**

* Supports various operations: `Add`, `Sub`, `Mult`, `Div`, `Pow`, `FloorDiv`, `BitAnd`, `BitOr`, `BitXor`, `LShift`, `RShift`.
* Handles nested tuples and lists using recursive calls with `get_current_value`.

**Notable Dependencies or Requirements:**

* `ast.Name`, `ast.Subscript`, `ast.Attribute`, `ast.Tuple`, `ast.List` ( Python AST classes)
* `evaluate_ast` function, not shown in this code snippet.
* `state.get`, `state.set_value` methods, not shown in this code snippet.

**Interfaces and Inheritance:**

None explicitly mentioned.

This summary aims to provide a clear and concise overview of the function's purpose, key implementation details, and notable dependencies.


key: chroma:document
string_value:
CODE:

def evaluate_boolop(node, state, static_tools, custom_tools):
    if isinstance(node.op, ast.And):
        for value in node.values:
            if not evaluate_ast(value, state, static_tools, custom_tools):
                return False
        return True
    elif isinstance(node.op, ast.Or):
        for value in node.values:
            if evaluate_ast(value, state, static_tools, custom_tools):
                return True


SUMMARY:
**Summary of `evaluate_boolop` Function**

Evaluates a Boolean expression represented as an Abstract Syntax Tree (AST) node in the context of a compilation or evaluation process.

**Primary Purpose**
Replaces a given Boolean operation AST node with its evaluated result, based on the values of its child nodes.

**Key Implementation Details**

* Uses recursive traversal to evaluate `And` and `Or` operations.
* For each operation, evaluates all child values using `evaluate_ast`.
* Returns `True` if any child is true (for `Or`) or all children are false (for `And`).
* Handles both `ast.And` and `ast.Or` operations.

**Notable Dependencies and Requirements**

* Uses Python's `ast` module for abstract syntax tree manipulation.
* Requires an instance of `ast.And` and/or `ast.Or` for evaluation.
* Accepts additional arguments: `state`, `static_tools`, and `custom_tools`.


key: chroma:document
string_value:
CODE:

def evaluate_binop(binop, state, static_tools, custom_tools):
    # Recursively evaluate the left and right operands
    left_val = evaluate_ast(binop.left, state, static_tools, custom_tools)
    right_val = evaluate_ast(binop.right, state, static_tools, custom_tools)

    # Determine the operation based on the type of the operator in the BinOp
    if isinstance(binop.op, ast.Add):
        return left_val + right_val
    elif isinstance(binop.op, ast.Sub):
        return left_val - right_val
    elif isinstance(binop.op, ast.Mult):
        return left_val * right_val
    elif isinstance(binop.op, ast.Div):
        return left_val / right_val
    elif isinstance(binop.op, ast.Mod):
        return left_val % right_val
    elif isinstance(binop.op, ast.Pow):
        return left_val**right_val
    elif isinstance(binop.op, ast.FloorDiv):
        return left_val // right_val
    elif isinstance(binop.op, ast.BitAnd):
        return left_val & right_val
    elif isinstance(binop.op, ast.BitOr):
        return left_val | right_val
    elif isinstance(binop.op, ast.BitXor):
        return left_val ^ right_val
    elif isinstance(binop.op, ast.LShift):
        return left_val << right_val
    elif isinstance(binop.op, ast.RShift):
        return left_val >> right_val
    else:


SUMMARY:
**Concise Summary**

`evaluate_binop` Evaluates binary operations on expression trees.

*   Primary Purpose: Applies recursive arithmetic evaluation to operands and performs the corresponding binary operation.
*   
*   Key Implementation Details:
    *   Recursively evaluates `left_val` and `right_val` using `evaluate_ast`.
    *   Uses type-specific operations based on `binop.op` (e.g., addition, subtraction).
*   
*   Notable Dependencies:
    *   `ast`: Abstract Syntax Tree module for parsing expression trees.
    *   `static_tools`, `custom_tools`: Optional custom toolboxes for static analysis.

**RAG System Notations:**

| Category | Description |
| --- | --- |
| **Function Signature** | `evaluate_binop(binop, state, static_tools, custom_tools)` |
| **Primary Purpose** | Evaluates binary operations on expression trees. |
| **Key Implementation Details** (`evaluate_binop`) |
| **Notable Dependencies** (External) |
|  • `ast` |
|  • `static_tools`, `custom_tools` |

Note: This summary is designed to be concise, readable, and easily consumable for developers within the RAG system.


key: chroma:document
string_value:
CODE:

def evaluate_assign(assign, state, static_tools, custom_tools):
    result = evaluate_ast(assign.value, state, static_tools, custom_tools)
    if len(assign.targets) == 1:
        target = assign.targets[0]
        set_value(target, result, state, static_tools, custom_tools)
    else:
        if len(assign.targets) != len(result):
            raise InterpreterError(f"Assign failed: expected {len(result)} values but got {len(assign.targets)}.")
        expanded_values = []
        for tgt in assign.targets:
            if isinstance(tgt, ast.Starred):
                expanded_values.extend(result)
            else:
                expanded_values.append(result)
        for tgt, val in zip(assign.targets, expanded_values):
            set_value(tgt, val, state, static_tools, custom_tools)


SUMMARY:
**Summary: `evaluate_assign` Function**

Evaluates an assignment expression and sets values to targets.

**Main Purpose and Functionality**

The `evaluate_assign` function evaluates an assignment expression in the AST by calling `evaluate_ast`. If there is only one target, it sets its value directly. Otherwise, it expands the result values based on the presence of starred targets and then sets values for each individual target.

**Key Implementation Details**

* Calls `evaluate_ast(assign.value, state, static_tools, custom_tools)`
* Handles single-target assignments with `set_value`
* Expands result values for starred targets using `expanded_values.extend(result)`

**Notable Dependencies or Requirements**

* `assign` and `result` references are used:
	+ `value`: AST node to be evaluated
	+ `targets`: tuple of target expressions
* External dependencies:
	+ `InterpreterError`
	+ `ast.Starred`
* Implemented interface:
	+ Uses `set_value` function, which is not shown in this code chunk.


key: chroma:document
string_value:
CODE:

def set_value(target, value, state, static_tools, custom_tools):
    if isinstance(target, ast.Name):
        if target.id in static_tools:
            raise InterpreterError(f"Cannot assign to name '{target.id}': doing this would erase the existing tool!")
        state[target.id] = value
    elif isinstance(target, ast.Tuple):
        if not isinstance(value, tuple):
            if hasattr(value, "__iter__") and not isinstance(value, (str, bytes)):
                value = tuple(value)
            else:
                raise InterpreterError("Cannot unpack non-tuple value")
        if len(target.elts) != len(value):
            raise InterpreterError("Cannot unpack tuple of wrong size")
        for i, elem in enumerate(target.elts):
            set_value(elem, value[i], state, static_tools, custom_tools)
    elif isinstance(target, ast.Subscript):
        obj = evaluate_ast(target.value, state, static_tools, custom_tools)
        key = evaluate_ast(target.slice, state, static_tools, custom_tools)
        obj[key] = value
    elif isinstance(target, ast.Attribute):
        obj = evaluate_ast(target.value, state, static_tools, custom_tools)


SUMMARY:
**Function Summary: `set_value`**

* **Purpose and Functionality:** Assigns values to targets in the AST using different assignment patterns (e.g., name assignments, tuple unpacking, slicing).
	+ Primary purpose: Simplify state management by providing a single interface for assigning values.
* **Key Implementation Details:**
	+
	- Handling `ast.Name`: checks if it's a tool being redefined, otherwise updates its value.
	- Handling `ast.Tuple` and `value`: performs tuple unpacking, catching errors on non-tuple inputs or mismatched lengths.
	- Handling `ast.Subscript` and slicing: evaluates the object and key using `evaluate_ast`, then assigns to it.
* **Notable Dependencies and Requirements:**
	+ Use of:
		- `ast.Name`
		- `ast.Tuple`
		- `ast.Subscript`
		- `ast.Attribute`
	+ External dependencies:
		- `InterpreterError` class (not shown in this code snippet)
	+ Interfaces and Classes used:
		- `evaluate_ast`: a shared utility function for evaluating AST nodes
		- `static_tools` dictionary (used to store tool configurations)


key: chroma:document
string_value:
CODE:

def evaluate_call(call, state, static_tools, custom_tools):
    if not (isinstance(call.func, ast.Attribute) or isinstance(call.func, ast.Name)):
        raise InterpreterError(f"This is not a correct function: {call.func}).")
    if isinstance(call.func, ast.Attribute):
        obj = evaluate_ast(call.func.value, state, static_tools, custom_tools)
        func_name = call.func.attr
        if not hasattr(obj, func_name):
            raise InterpreterError(f"Object {obj} has no attribute {func_name}")
        func = getattr(obj, func_name)

    elif isinstance(call.func, ast.Name):
        func_name = call.func.id
        if func_name in state:
            func = state[func_name]
        elif func_name in static_tools:
            func = static_tools[func_name]
        elif func_name in custom_tools:
            func = custom_tools[func_name]
        elif func_name in ERRORS:
            func = ERRORS[func_name]
        else:
            raise InterpreterError(
                f"It is not permitted to evaluate other functions than the provided tools or functions defined in previous code (tried to execute {call.func.id})."
            )

    args = []
    for arg in call.args:
        if isinstance(arg, ast.Starred):
            args.extend(evaluate_ast(arg.value, state, static_tools, custom_tools))
        else:
            args.append(evaluate_ast(arg, state, static_tools, custom_tools))

    args = []
    for arg in call.args:
        if isinstance(arg, ast.Starred):
            unpacked = evaluate_ast(arg.value, state, static_tools, custom_tools)
            if not hasattr(unpacked, "__iter__") or isinstance(unpacked, (str, bytes)):
                raise InterpreterError(f"Cannot unpack non-iterable value {unpacked}")
            args.extend(unpacked)
        else:
            args.append(evaluate_ast(arg, state, static_tools, custom_tools))

    kwargs = {keyword.arg: evaluate_ast(keyword.value, state, static_tools, custom_tools) for keyword in call.keywords}

    if isinstance(func, type) and len(func.__module__.split(".")) > 1:  # Check for user-defined classes
        # Instantiate the class using its constructor
        obj = func.__new__(func)  # Create a new instance of the class
        if hasattr(obj, "__init__"):  # Check if the class has an __init__ method
            obj.__init__(*args, **kwargs)  # Call the __init__ method correctly
        return obj
    else:
        if func_name == "super":
            if not args:
                if "__class__" in state and "self" in state:
                    return super(state["__class__"], state["self"])
                else:
                    raise InterpreterError("super() needs at least one argument")
            cls = args[0]
            if not isinstance(cls, type):
                raise InterpreterError("super() argument 1 must be type")
            if len(args) == 1:
                return super(cls)
            elif len(args) == 2:
                instance = args[1]
                return super(cls, instance)
            else:
                raise InterpreterError("super() takes at most 2 arguments")
        else:
            if func_name == "print":
                output = " ".join(map(str, args))
                global PRINT_OUTPUTS
                PRINT_OUTPUTS += output + "\n"
                # cap the number of lines
                return None
            else:  # Assume it's a callable object
                output = func(*args, **kwargs)


SUMMARY:
**Function Summary: evaluate_call**

Evaluates a function call by resolving its attributes and arguments.

### Main Purpose and Functionality

Resolves a function call by checking if its `func` attribute is an `ast.Attribute` or `ast.Name`. If it's an `ast.Attribute`, it calls another `evaluate_ast` function with the object and its attribute. If it's an `ast.Name`, it looks up the corresponding function name in the provided `state`, `static_tools`, `custom_tools`, and a hardcoded dictionary `ERRORS`.

### Key Implementation Details

* Resolves attribute by checking if it exists on the object.
* Instructs error handling using Python built-in errors (`InterpreterError`) for incorrect function calls or missing attributes.
* Handles starred arguments by calling `evaluate_ast` recursively.

### Notable Dependencies and Requirements

* `ast`: For parsing abstract syntax tree nodes (AST) from call objects.
* `static_tools`, `custom_tools`, and `ERRORS`: External dictionaries containing resolved functions with their respective names.
* `PRINT_OUTPUTS`: A global variable to accumulate print output values, capped at a certain lines.

### Method Signatures and Interfaces

The function uses the following method signatures:
```python
def evaluate_ast(...)
```
- Resolves AST attributes.
- Evaluates arguments using `evaluate_ast`.
- Handles starred arguments using recursive calls to `evaluate_ast`.

This summary provides an overview of the main purpose, key implementation details, notable dependencies, and notable functions used in this code snippet.


key: chroma:document
string_value:
CODE:

def evaluate_subscript(subscript, state, static_tools, custom_tools):
    index = evaluate_ast(subscript.slice, state, static_tools, custom_tools)
    value = evaluate_ast(subscript.value, state, static_tools, custom_tools)

    if isinstance(value, str) and isinstance(index, str):
        raise InterpreterError("You're trying to subscript a string with a string index, which is impossible")
    if isinstance(value, pd.core.indexing._LocIndexer):
        parent_object = value.obj
        return parent_object.loc[index]
    if isinstance(value, (pd.DataFrame, pd.Series, np.ndarray)):
        return value[index]
    elif isinstance(value, pd.core.groupby.generic.DataFrameGroupBy):
        return value[index]
    elif isinstance(index, slice):
        return value[index]
    elif isinstance(value, (list, tuple)):
        if not (-len(value) <= index < len(value)):
            raise InterpreterError(f"Index {index} out of bounds for list of length {len(value)}")
        return value[int(index)]
    elif isinstance(value, str):
        if not (-len(value) <= index < len(value)):
            raise InterpreterError(f"Index {index} out of bounds for string of length {len(value)}")
        return value[index]
    elif index in value:
        return value[index]
    elif isinstance(index, str) and isinstance(value, Mapping):
        close_matches = difflib.get_close_matches(index, list(value.keys()))
        if len(close_matches) > 0:
            return value[close_matches[0]]


SUMMARY:
**Function Summary: `evaluate_subscript`**

* **Primary Purpose:** Evaluates an subscript operation on various data structures (pandas Series, DataFrames, NumPy arrays, lists, and dictionaries).

**Key Implementation Details:**

* Utilizes separate evaluations for `slice` and `value` arguments using `evaluate_ast`.
* Handles different types of values, including strings, pandas objects, arrays, lists, and mappings.
* Implements bounds checking for array indices and string substrings.
* Uses difflib to find close matches in dictionary keys.

**Notable Dependencies and Requirements:**

* `pd`: Pandas library
* `np`: NumPy library
* `difflib`: Python standard library


key: chroma:document
string_value:
CODE:

def evaluate_name(name, state, static_tools, custom_tools):
    if name.id in state:
        return state[name.id]
    elif name.id in static_tools:
        return static_tools[name.id]
    elif name.id in ERRORS:
        return ERRORS[name.id]
    close_matches = difflib.get_close_matches(name.id, list(state.keys()))
    if len(close_matches) > 0:
        return state[close_matches[0]]


SUMMARY:
**Function Summary:**
```python
evaluate_name(date_time, state, static_tools, custom_tools):
    Retrieves the value of a named variable from local/state/external sources.

    **Purpose:** Returns the value of a variable with the given `name.id`.

    **Key Details:**

    * Checks local state (local variables) for matches
    * Fallbacks to external static tools if local state is empty
    * If both local and static tools are exhausted, raises an InterpreterError

    **External Dependencies:**
        + `difflib.get_close_matches(name.id, list(state.keys()))` (library dependency)
```
This summary provides a concise overview of the function's purpose and key implementation details. It highlights that the function is designed to retrieve variable values from multiple sources (state, static tools, and external sources) before raising an error if no match is found. The dependencies are explicitly mentioned, including the `difflib` library used for close matches detection.


key: chroma:document
string_value:
CODE:

def evaluate_condition(condition, state, static_tools, custom_tools):
    left = evaluate_ast(condition.left, state, static_tools, custom_tools)
    comparators = [evaluate_ast(c, state, static_tools, custom_tools) for c in condition.comparators]
    ops = [type(op) for op in condition.ops]

    result = True
    current_left = left

    for op, comparator in zip(ops, comparators):
        if op == ast.Eq:
            current_result = current_left == comparator
        elif op == ast.NotEq:
            current_result = current_left != comparator
        elif op == ast.Lt:
            current_result = current_left < comparator
        elif op == ast.LtE:
            current_result = current_left <= comparator
        elif op == ast.Gt:
            current_result = current_left > comparator
        elif op == ast.GtE:
            current_result = current_left >= comparator
        elif op == ast.Is:
            current_result = current_left is comparator
        elif op == ast.IsNot:
            current_result = current_left is not comparator
        elif op == ast.In:
            current_result = current_left in comparator
        elif op == ast.NotIn:
            current_result = current_left not in comparator
        else:
            raise InterpreterError(f"Operator not supported: {op}")

        result = result & current_result
        current_left = comparator

        if isinstance(result, bool) and not result:
            break



SUMMARY:
**Summary**

The `evaluate_condition` function assesses the truth value of an AST (Abstract Syntax Tree) condition by recursively evaluating its constituent parts. It takes into account various comparison operators and logical operations.

**Key Implementation Details**

* Evaluates left-hand side expression (`left`) using `evaluate_ast`
* Recursively evaluates comparators and applies corresponding operators to `current_left` and comparator
* Performs element-wise AND operation on results of each comparison
* Returns final result as a boolean value or Pandas Series if it is a boolean

**Key Algorithm/Design Choice**

* Uses dictionary-based mapping of operator types to their corresponding AST nodes

**Notable Dependencies/Requirements**

None mentioned in the code. However, `ast` (Abstract Syntax Tree) module and potentially `pd.Series` (Pandas Series) library are dependencies.

**Method Signatures and Interfaces**

- `evaluate_condition(condition, state, static_tools, custom_tools)`
  - Evaluates AST condition

- `evaluate_ast(node, state, static_tools, custom_tools)`
  - Evaluates abstract syntax tree node


key: chroma:document
string_value:
CODE:

def evaluate_if(if_statement, state, static_tools, custom_tools):
    result = None
    test_result = evaluate_ast(if_statement.test, state, static_tools, custom_tools)
    if test_result:
        for line in if_statement.body:
            line_result = evaluate_ast(line, state, static_tools, custom_tools)
            if line_result is not None:
                result = line_result
    else:
        for line in if_statement.orelse:
            line_result = evaluate_ast(line, state, static_tools, custom_tools)
            if line_result is not None:
                result = line_result


SUMMARY:
**Function Summary: `evaluate_if`**

Evaluates the condition and body of an `if` statement using recursive `evaluate_ast`. Returns the evaluation result.

**Main Purpose and Functionality**

This function takes an `if` statement, current state, static tools, and custom tools as input. It:

1. Evaluates the test condition using `evaluate_ast`.
2. If the condition is true:
	* Iterates through the body of the statement (`for line in if_statement.body`) and evaluates each line using `evaluate_ast`.
	* Collects the result from the first non-None evaluation.
3. If the condition is false:
	* Iterates through the optional `orelse` clause (if present) and evaluates each line using `evaluate_ast`.
	* Collects the result from the first non-None evaluation.

**Key Implementation Details**

* Uses `evaluate_ast` function recursively to evaluate statements.
* Checks for `is not None` in evaluation results to differentiate between successful and unsuccessful evaluations.

**Notable Dependencies or Requirements**

* External dependency: `evaluate_ast` function, which is not shown in this snippet.


key: chroma:document
string_value:
CODE:

def evaluate_for(for_loop, state, static_tools, custom_tools):
    result = None
    iterator = evaluate_ast(for_loop.iter, state, static_tools, custom_tools)
    for counter in iterator:
        set_value(for_loop.target, counter, state, static_tools, custom_tools)
        for node in for_loop.body:
            try:
                line_result = evaluate_ast(node, state, static_tools, custom_tools)
                if line_result is not None:
                    result = line_result
            except BreakException:
                break
            except ContinueException:
                continue
        else:
            continue
        break


SUMMARY:
**Summary:**

`evaluate_for` function evaluates the functionality of a nested for loop (`for_loop`).

**Primary Purpose:**
The main purpose is to execute the body of the for loop and its inner loops while handling exceptions, setting values in the target variable.

**Key Implementation Details:**

*   Iterates over an iterator containing counters from `evaluate_ast(for_loop.iter, ...)`.
*   Sets the target variable's value using `set_value` (on each counter iteration).
*   Evaluates nested nodes within `for_loop.body` using `evaluate_ast` and accumulates results.

**Notable Dependencies or Requirements:**

*   `for_loop`: for loop object with attributes `.iter`, `.target`, and `.body`.
*   `state`, `static_tools`, and `custom_tools`: objects passed to `evaluate_for` which are used for state management.
*   `BreakException` and `ContinueException`: custom exception classes (not shown in the provided code snippet).
*   External dependencies on `evaluate_ast` function.


key: chroma:document
string_value:
CODE:

def evaluate_listcomp(listcomp, state, static_tools, custom_tools):
    def inner_evaluate(generators, index, current_state):
        if index >= len(generators):
            return [evaluate_ast(listcomp.elt, current_state, static_tools, custom_tools)]
        generator = generators[index]
        iter_value = evaluate_ast(generator.iter, current_state, static_tools, custom_tools)
        result = []
        for value in iter_value:
            new_state = current_state.copy()
            if isinstance(generator.target, ast.Tuple):
                for idx, elem in enumerate(generator.target.elts):
                    new_state[elem.id] = value[idx]
            else:
                new_state[generator.target.id] = value
            if all(evaluate_ast(if_clause, new_state, static_tools, custom_tools) for if_clause in generator.ifs):
                result.extend(inner_evaluate(generators, index + 1, new_state))
        return result



SUMMARY:
**Evaluation of List Comprehensions Functionality**

The `evaluate_listcomp` function evaluates a list comprehension (`listcomp`) and returns the resulting list.

**Key Functionality:**

*   Iterates through a sequence of generators passed as an argument.
*   Applies transformations on each generator output using `eval_ast`.
*   Checks for valid conditions using `eval_ast` (if-elif-else clauses).
*   Evaluates the final result by applying state updates to new states.

**Key Implementation Details:**

*   Uses recursion (`inner_evaluate`) to iterate through generators.
*   Appends intermediate results to a list only if all conditionals evaluate as true.

**Notable Dependencies or Requirements:**

*   `listcomp.elt`: The element being processed in the iteration.
*   `state`: The current state being updated.
*   `static_tools` and `custom_tools`: Functions for performing operations on states.
*   External dependency on `ast.Tuple`, `ast`, and other related classes.

**Interface Involvement:**

*   Extends a list with recursive results from the inner function (`inner_evaluate`).
*   Interfaces with `eval_ast` to perform state updates and conditional checks.


key: chroma:document
string_value:
CODE:

def evaluate_try(try_node, state, static_tools, custom_tools):
    try:
        for stmt in try_node.body:
            evaluate_ast(stmt, state, static_tools, custom_tools)
    except Exception as e:
        matched = False
        for handler in try_node.handlers:
            if handler.type is None or isinstance(e, evaluate_ast(handler.type, state, static_tools, custom_tools)):
                matched = True
                if handler.name:
                    state[handler.name] = e
                for stmt in handler.body:
                    evaluate_ast(stmt, state, static_tools, custom_tools)
                break
        if not matched:
            raise e
    else:
        if try_node.orelse:
            for stmt in try_node.orelse:
                evaluate_ast(stmt, state, static_tools, custom_tools)
    finally:
        if try_node.finalbody:
            for stmt in try_node.finalbody:


SUMMARY:
**Function Summary**

`evaluate_try` evaluates the semantic scope of a `try` node within an abstract syntax tree (AST) under the provided `state`, using both `static_tools` and `custom_tools`. The function attempts to:

* Evaluate statements in the try block (`body`) before potentially catching exceptions.
* Catch exceptions in handlers with matching types or values from previous expressions.
* Handle successful execution without raising exceptions.

**Key Implementation Details**

* Iterates through:
	+ Statements in `try_node.body` using `evaluate_ast`.
	+ Handlers in `try_node.handlers` if exceptions aren't matched.
	+ Statements in `try_node.orelse` (alternative) block if no exception is raised.
* Uses conditional statements to handle successful execution or exceptions.

**Notable Dependencies and Requirements**

* Uses `evaluate_ast` function, which is not defined in this snippet but likely evaluates individual AST nodes.
* References:
	+ `handlers.type`
	+ `handler.name`
	+ `custom_tools` and `static_tools` (no specific interface provided).
* Includes external dependencies on `Exception` and its `as_e` method.


key: chroma:document
string_value:
CODE:

def evaluate_raise(raise_node, state, static_tools, custom_tools):
    if raise_node.exc is not None:
        exc = evaluate_ast(raise_node.exc, state, static_tools, custom_tools)
    else:
        exc = None
    if raise_node.cause is not None:
        cause = evaluate_ast(raise_node.cause, state, static_tools, custom_tools)
    else:
        cause = None
    if exc is not None:
        if cause is not None:
            raise exc from cause
        else:
            raise exc
    else:


SUMMARY:
**Summary: Re-Raise Function**

The `evaluate_raise` function reraises an exception by propagating its cause. It takes four arguments:

* `raise_node`: The node representing the re-raise operation
* `state`: The program execution state (not used in this implementation)
* `static_tools`, `custom_tools`: External tool instances for static analysis and custom functionality

**Primary Purpose:** Re-raises an exception with its cause.

**Key Implementation Details:**

* Evaluates the exception's cause using `evaluate_ast`
* Reraises the exception using `raise` syntax, optionally with a `from` clause to propagate the cause
* Handles cases where exceptions are not present or active

**Notable Dependencies/Requirements:**

* `InterpreterError` class from external libraries or modules (not specified in this code snippet)
* External tool instances (`static_tools`, `custom_tools`) for static analysis and custom functionality


key: chroma:document
string_value:
CODE:

def evaluate_assert(assert_node, state, static_tools, custom_tools):
    test_result = evaluate_ast(assert_node.test, state, static_tools, custom_tools)
    if not test_result:
        if assert_node.msg:
            msg = evaluate_ast(assert_node.msg, state, static_tools, custom_tools)
            raise AssertionError(msg)
        else:
            # Include the failing condition in the assertion message
            test_code = ast.unparse(assert_node.test)


SUMMARY:
**Concise Summary**

Evaluates an assertion statement (`evaluate_assert`) that checks if a given PHP code snippet passes or fails. If validation fails:

* Performs dynamic analysis on the test source code using `static_tools` and raises an `AssertionError`.
* Uses a nested call to recursively evaluate custom validation logic stored in `custom_tools`.
* If no external AST evaluation is available (i.e., `test_node.msg` is empty), it includes the PHP code snippet being asserted in the error message.

**Key Implementation Details**

* Evaluates test source code using `evaluate_ast(assert_node.test, state, static_tools, custom_tools)`
* Performs dynamic analysis on the test source code
* Uses `EvaluateAST` function for validation
* Involves nested call to recursively evaluate custom validation logic (`custom_tools`)

**External Dependencies/Requirements**

* No external dependencies or imports
* Reuses `evaluate_ast` and `ast.unparse` functions
* `state`, `static_tools`, and `custom_tools` objects/variables expected as inputs


key: chroma:document
string_value:
CODE:

def evaluate_with(with_node, state, static_tools, custom_tools):
    contexts = []
    for item in with_node.items:
        context_expr = evaluate_ast(item.context_expr, state, static_tools, custom_tools)
        if item.optional_vars:
            state[item.optional_vars.id] = context_expr.__enter__()
            contexts.append(state[item.optional_vars.id])
        else:
            context_var = context_expr.__enter__()
            contexts.append(context_var)

    try:
        for stmt in with_node.body:
            evaluate_ast(stmt, state, static_tools, custom_tools)
    except Exception as e:
        for context in reversed(contexts):
            context.__exit__(type(e), e, e.__traceback__)
        raise
    else:
        for context in reversed(contexts):


SUMMARY:
Here is the concise and focused code summary:

**Functionality:** `evaluate_with` evaluates a Python with statement node in the Abstract Syntax Tree (AST) using provided tools and state.

**Main Purpose:**

The primary purpose of this function is to execute the statements within a with block, handling both optional and required variables.

**Key Implementation Details:**

* Evaluate the context expression for each item in `with_node.items` and create a context or variable depending on `item.optional_vars`.
* Iterate through `with_node.body` statements, evaluating them using `evaluate_ast`. Catch any exceptions, exiting contexts if applicable.
* After executing all statements, exit any remaining contexts.

**Dependencies:**

None explicitly listed. However, references to:

* `state`: an object used to manage session state (inherited)
* `static_tools`, `custom_tools`: custom evaluation tools (custom implementation)

`evaluate_ast` is a helper function not shown in the provided code snippet.


key: chroma:document
string_value:
CODE:

def import_modules(expression, state, authorized_imports):
    def check_module_authorized(module_name):
        module_path = module_name.split(".")
        module_subpaths = [".".join(module_path[:i]) for i in range(1, len(module_path) + 1)]
        return any(subpath in authorized_imports for subpath in module_subpaths)

    if isinstance(expression, ast.Import):
        for alias in expression.names:
            if check_module_authorized(alias.name):
                module = import_module(alias.name)
                state[alias.asname or alias.name] = module
            else:
                raise InterpreterError(
                    f"Import of {alias.name} is not allowed. Authorized imports are: {str(authorized_imports)}"
                )
        return None
    elif isinstance(expression, ast.ImportFrom):
        if check_module_authorized(expression.module):
            module = __import__(expression.module, fromlist=[alias.name for alias in expression.names])
            for alias in expression.names:
                state[alias.asname or alias.name] = getattr(module, alias.name)
        else:
            raise InterpreterError(f"Import from {expression.module} is not allowed.")


SUMMARY:
**Summary: `import_modules` Function**

**Purpose and Functionality**
The `import_modules` function filters imports in an AST expression based on authorized module names. It verifies whether the imported modules are part of the `authorized_imports` list, raising an error if not.

**Key Implementation Details**

* The function uses the `ast.Import` and `ast.ImportFrom` types to process import expressions.
* A separate check is performed for each alias in both `Import` and `ImportFrom` expressions.
* For `ImportFrom` expressions, it iterates over aliases and assigns attribute references (`getattr`) from the imported modules.

**Notable Dependencies or Requirements**

* Uses `ast.Import`, `ast.ImportFrom`, `import_module`, and `__import__` functions from the Python interpreter.
* References an unbound variable `authorized_imports`.
* No external dependencies other than the standard library. 

This summary should be sufficient for use in a RAG system, providing essential context for understanding this code chunk without excessive details.


key: chroma:document
string_value:
CODE:

def evaluate_dictcomp(dictcomp, state, static_tools, custom_tools):
    result = {}
    for gen in dictcomp.generators:
        iter_value = evaluate_ast(gen.iter, state, static_tools, custom_tools)
        for value in iter_value:
            new_state = state.copy()
            set_value(gen.target, value, new_state, static_tools, custom_tools)
            if all(evaluate_ast(if_clause, new_state, static_tools, custom_tools) for if_clause in gen.ifs):
                key = evaluate_ast(dictcomp.key, new_state, static_tools, custom_tools)
                val = evaluate_ast(dictcomp.value, new_state, static_tools, custom_tools)
                result[key] = val


SUMMARY:
**Summary: Evaluation of Dictionary Computation**

**Purpose and Functionality:** Evaluates a dictionary comprehension (dictcomp) given its specification (`generators`, `ifs`, `key`, and `value`), a state, and toolkits.

* Iterates over generator-producing values
* Applies conditions specified in if-clauses for each value
* If all conditions true, computes key-value pair based on provided expressions

### Key Implementation Details

* Utilizes `evaluate_ast` to parse and evaluate AST nodes (`iter`, `target`, `if_clause`, `key`, `value`)
* Iterates over generator-producing values using nested loops

### Notable Dependencies and Requirements

* `set_value`: utility function not shown; likely computes value based on provided expressions
* `evaluate_ast`: takes an AST node, state, tools, and returns the evaluated result
* `state`: manages program state, e.g., data used in evaluation
* `static_tools` and `custom_tools`: external toolkit collections (implied by their names)

This summary provides a concise overview of the function's purpose, key implementation details, and notable dependencies.


key: chroma:document
string_value:
CODE:

def evaluate_ast(
    expression: ast.AST,
    state: Dict[str, Any],
    static_tools: Dict[str, Callable],
    custom_tools: Dict[str, Callable],
    authorized_imports: List[str] = LIST_SAFE_MODULES,
):
    """
    Evaluate an abstract syntax tree using the content of the variables stored in a state and only evaluating a given
    set of functions.

    This function will recurse trough the nodes of the tree provided.

    Args:
        expression (`ast.AST`):
            The code to evaluate, as an abstract syntax tree.
        state (`Dict[str, Any]`):
            A dictionary mapping variable names to values. The `state` is updated if need be when the evaluation
            encounters assignements.
        static_tools (`Dict[str, Callable]`):
            Functions that may be called during the evaluation. Trying to change one of these static_tools will raise an error.
        custom_tools (`Dict[str, Callable]`):
            Functions that may be called during the evaluation. These static_tools can be overwritten.
        authorized_imports (`List[str]`):
            The list of modules that can be imported by the code. By default, only a few safe modules are allowed.
            Add more at your own risk!
    """
    global OPERATIONS_COUNT
    if OPERATIONS_COUNT >= MAX_OPERATIONS:
        raise InterpreterError(
            f"Reached the max number of operations of {MAX_OPERATIONS}. Maybe there is an infinite loop somewhere in the code, or you're just asking too many calculations."
        )
    OPERATIONS_COUNT += 1
    if isinstance(expression, ast.Assign):
        # Assignement -> we evaluate the assignment which should update the state
        # We return the variable assigned as it may be used to determine the final result.
        return evaluate_assign(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.AugAssign):
        return evaluate_augassign(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Call):
        # Function call -> we return the value of the function call
        return evaluate_call(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Constant):
        # Constant -> just return the value
        return expression.value
    elif isinstance(expression, ast.Tuple):
        return tuple(evaluate_ast(elt, state, static_tools, custom_tools) for elt in expression.elts)
    elif isinstance(expression, (ast.ListComp, ast.GeneratorExp)):
        return evaluate_listcomp(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.UnaryOp):
        return evaluate_unaryop(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Starred):
        return evaluate_ast(expression.value, state, static_tools, custom_tools)
    elif isinstance(expression, ast.BoolOp):
        # Boolean operation -> evaluate the operation
        return evaluate_boolop(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Break):
        raise BreakException()
    elif isinstance(expression, ast.Continue):
        raise ContinueException()
    elif isinstance(expression, ast.BinOp):
        # Binary operation -> execute operation
        return evaluate_binop(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Compare):
        # Comparison -> evaluate the comparison
        return evaluate_condition(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Lambda):
        return evaluate_lambda(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.FunctionDef):
        return evaluate_function_def(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Dict):
        # Dict -> evaluate all keys and values
        keys = [evaluate_ast(k, state, static_tools, custom_tools) for k in expression.keys]
        values = [evaluate_ast(v, state, static_tools, custom_tools) for v in expression.values]
        return dict(zip(keys, values))
    elif isinstance(expression, ast.Expr):
        # Expression -> evaluate the content
        return evaluate_ast(expression.value, state, static_tools, custom_tools)
    elif isinstance(expression, ast.For):
        # For loop -> execute the loop
        return evaluate_for(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.FormattedValue):
        # Formatted value (part of f-string) -> evaluate the content and return
        return evaluate_ast(expression.value, state, static_tools, custom_tools)
    elif isinstance(expression, ast.If):
        # If -> execute the right branch
        return evaluate_if(expression, state, static_tools, custom_tools)
    elif hasattr(ast, "Index") and isinstance(expression, ast.Index):
        return evaluate_ast(expression.value, state, static_tools, custom_tools)
    elif isinstance(expression, ast.JoinedStr):
        return "".join([str(evaluate_ast(v, state, static_tools, custom_tools)) for v in expression.values])
    elif isinstance(expression, ast.List):
        # List -> evaluate all elements
        return [evaluate_ast(elt, state, static_tools, custom_tools) for elt in expression.elts]
    elif isinstance(expression, ast.Name):
        # Name -> pick up the value in the state
        return evaluate_name(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Subscript):
        # Subscript -> return the value of the indexing
        return evaluate_subscript(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.IfExp):
        test_val = evaluate_ast(expression.test, state, static_tools, custom_tools)
        if test_val:
            return evaluate_ast(expression.body, state, static_tools, custom_tools)
        else:
            return evaluate_ast(expression.orelse, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Attribute):
        value = evaluate_ast(expression.value, state, static_tools, custom_tools)
        return getattr(value, expression.attr)
    elif isinstance(expression, ast.Slice):
        return slice(
            evaluate_ast(expression.lower, state, static_tools, custom_tools)
            if expression.lower is not None
            else None,
            evaluate_ast(expression.upper, state, static_tools, custom_tools)
            if expression.upper is not None
            else None,
            evaluate_ast(expression.step, state, static_tools, custom_tools) if expression.step is not None else None,
        )
    elif isinstance(expression, ast.DictComp):
        return evaluate_dictcomp(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.While):
        return evaluate_while(expression, state, static_tools, custom_tools)
    elif isinstance(expression, (ast.Import, ast.ImportFrom)):
        return import_modules(expression, state, authorized_imports)
    elif isinstance(expression, ast.ClassDef):
        return evaluate_class_def(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Try):
        return evaluate_try(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Raise):
        return evaluate_raise(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Assert):
        return evaluate_assert(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.With):
        return evaluate_with(expression, state, static_tools, custom_tools)
    elif isinstance(expression, ast.Set):
        return {evaluate_ast(elt, state, static_tools, custom_tools) for elt in expression.elts}
    elif isinstance(expression, ast.Return):
        raise ReturnException(
            evaluate_ast(expression.value, state, static_tools, custom_tools) if expression.value else None
        )
    else:
        # For now we refuse anything else. Let's add things as we need them.


SUMMARY:
**Summary**

The `evaluate_ast` function is a recursive interpreter for Python abstract syntax trees (ASTs). It evaluates the syntax tree using values stored in a state dictionary and only accesses authorized custom functions.

**Main Purpose**
Evaluates a Python AST, updating the state dictionary accordingly.

**Key Implementation Details**

* Recursively traverses the AST nodes
* Handles various node types, including expressions, statements, control structures, imports, and exceptions
* Uses dictionaries to store and retrieve values in the state

**Notable Dependencies or Requirements**

* `ast` module for working with abstract syntax trees
* Custom functions (`static_tools` and `custom_tools`) can be passed as arguments
* Authorized imports (`authorized_imports`) list ensures controlled access to external modules

**Summary Function Signature**
```python
def evaluate_ast(
    expression: ast.AST,
    state: Dict[str, Any],
    static_tools: Dict[str, Callable],
    custom_tools: Dict[str, Callable],
    authorized_imports: List[str] = LIST_SAFE_MODULES
):
```

**Example Output Types**

* Evaluated expressions with values (e.g., integers, strings)
* Updated state dictionary after evaluating assignments
* None or exception raised for unsupported types


key: chroma:document
string_value:
CODE:

def truncate_print_outputs(print_outputs: str, max_len_outputs: int = MAX_LEN_OUTPUT) -> str:
    if len(print_outputs) < max_len_outputs:
        return print_outputs
    else:


SUMMARY:
**Function Summary: `truncate_print_outputs`**

The `truncate_print_outputs` function truncates the input `print_outputs` string to a maximum length of `max_len_outputs`. If the length exceeds the limit, it returns a formatted string displaying the original output and the truncation message.

**Key Features:**

*   Takes an optional `max_len_outputs` parameter with default value `MAX_LEN_OUTPUT`.
*   Returns an updated string if the length check passes; otherwise, includes a truncation message.

**Implementation Details:**

*   Uses slicing (`print_outputs[:max_len_outputs]`) to truncate the input string.
*   Formats the output into a single string with the truncated text and a descriptive message.


key: chroma:document
string_value:
CODE:

def evaluate_python_code(
    code: str,
    static_tools: Optional[Dict[str, Callable]] = None,
    custom_tools: Optional[Dict[str, Callable]] = None,
    state: Optional[Dict[str, Any]] = None,
    authorized_imports: List[str] = LIST_SAFE_MODULES,
):
    """
    Evaluate a python expression using the content of the variables stored in a state and only evaluating a given set
    of functions.

    This function will recurse through the nodes of the tree provided.

    Args:
        code (`str`):
            The code to evaluate.
        static_tools (`Dict[str, Callable]`):
            The functions that may be called during the evaluation.
            These tools cannot be overwritten in the code: any assignment to their name will raise an error.
        custom_tools (`Dict[str, Callable]`):
            The functions that may be called during the evaluation.
            These tools can be overwritten in the code: any assignment to their name will overwrite them.
        state (`Dict[str, Any]`):
            A dictionary mapping variable names to values. The `state` should contain the initial inputs but will be
            updated by this function to contain all variables as they are evaluated.
            The print outputs will be stored in the state under the key 'print_outputs'.
    """
    try:
        expression = ast.parse(code)
    except SyntaxError as e:
        raise SyntaxError(f"The code generated by the agent is not valid.\n{e}")
    if state is None:
        state = {}
    if static_tools is None:
        static_tools = {}
    if custom_tools is None:
        custom_tools = {}
    result = None
    global PRINT_OUTPUTS
    PRINT_OUTPUTS = ""
    global OPERATIONS_COUNT
    OPERATIONS_COUNT = 0
    try:
        for node in expression.body:
            result = evaluate_ast(node, state, static_tools, custom_tools, authorized_imports)
        state["print_outputs"] = truncate_print_outputs(PRINT_OUTPUTS, max_len_outputs=MAX_LEN_OUTPUT)
        return result
    except InterpreterError as e:
        msg = truncate_print_outputs(PRINT_OUTPUTS, max_len_outputs=MAX_LEN_OUTPUT)
        msg += f"EXECUTION FAILED:\nEvaluation stopped at line '{ast.get_source_segment(code, node)}' because of the following error:\n{e}"


SUMMARY:
**Function Summary:**

`evaluate_python_code`

Evaluates a Python expression by parsing its AST and traversing its nodes.

**Purpose and Functionality:**

Evaluate a Python code snippet using the provided `static_tools`, `custom_tools`, and `state`. Recursively traverse the AST, updating the `state` with evaluated variables.

**Key Implementation Details:**

* Parses the input `code` using `ast.parse()`
* Iterates through each node in the parsed AST
* Calls `evaluate_ast()` for each node, passing the updated `state`, `static_tools`, and `custom_tools`
* Stores print outputs in the `state`

**Notable Dependencies and Requirements:**

* `ast`: Used to parse Python code into an Abstract Syntax Tree (AST)
* `globals`: Accessed using `global`
* `print` output: Storied in `PRINT_OUTPUTS`
* External libraries like `ast` are assumed to be required

**Method Signatures and Interfaces:**

* `evaluate_ast(node, state, static_tools, custom_tools, authorized_imports)`: Evaluated recursively for each AST node
* `truncate_print_outputs(output, max_len_outputs=MAX_LEN_OUTPUT)`: Truncates print outputs to a limited length


key: chroma:document
string_value:
CODE:

def pull_message(step_log: dict, test_mode: bool = True):
    try:
        from gradio import ChatMessage
    except ImportError:
        if test_mode:

            class ChatMessage:
                def __init__(self, role, content, metadata=None):
                    self.role = role
                    self.content = content
                    self.metadata = metadata
        else:
            raise ImportError("Gradio should be installed in order to launch a gradio demo.")

    if step_log.get("rationale"):
        yield ChatMessage(role="assistant", content=step_log["rationale"])
    if step_log.get("tool_call"):
        used_code = step_log["tool_call"]["tool_name"] == "code interpreter"
        content = step_log["tool_call"]["tool_arguments"]
        if used_code:
            content = f"```py\n{content}\n```"
        yield ChatMessage(
            role="assistant",
            metadata={"title": f"🛠️ Used tool {step_log['tool_call']['tool_name']}"},
            content=str(content),
        )
    if step_log.get("observation"):
        yield ChatMessage(role="assistant", content=f"```\n{step_log['observation']}\n```")
    if step_log.get("error"):
        yield ChatMessage(
            role="assistant",
            content=str(step_log["error"]),
            metadata={"title": "💥 Error"},


SUMMARY:
**Function Summary: `pull_message`**

**Purpose and Functionality:**
This function generates and yields Gradio ChatMessage objects based on the input `step_log` dictionary. It provides context-specific messages for demo purposes, including tool usage, observations, errors, and rationales.

**Key Implementation Details:**

* Uses a try-except block to handle import errors with optional test mode fallback.
* Defines a mock `ChatMessage` class in test mode if Gradio is not installed.
* Yields ChatMessage objects with structured content and metadata, including tool usage information.
* Handles specific log key values (rationale, tool_call, observation, error) and generates corresponding messages.

**Notable Dependencies:**

* `gradio`: Required for the function to work properly in demo mode.
* External dependencies:
	+ `Gradio` module is loaded dynamically on import.

**Method Signatures and Interfaces:**
The function uses the `yield` keyword to generate a sequence of ChatMessage objects, which can be consumed by other parts of the program.


key: chroma:document
string_value:
CODE:

def stream_to_gradio(agent, task: str, test_mode: bool = False, **kwargs):
    """Runs an agent with the given task and streams the messages from the agent as gradio ChatMessages."""

    try:
        from gradio import ChatMessage
    except ImportError:
        if test_mode:

            class ChatMessage:
                def __init__(self, role, content, metadata=None):
                    self.role = role
                    self.content = content
                    self.metadata = metadata
        else:
            raise ImportError("Gradio should be installed in order to launch a gradio demo.")

    for step_log in agent.run(task, stream=True, **kwargs):
        if isinstance(step_log, dict):
            for message in pull_message(step_log, test_mode=test_mode):
                yield message

    final_answer = step_log  # Last log is the run's final_answer

    if isinstance(final_answer, AgentText):
        yield ChatMessage(role="assistant", content=f"**Final answer:**\n```\n{final_answer.to_string()}\n```")
    elif isinstance(final_answer, AgentImage):
        yield ChatMessage(
            role="assistant",
            content={"path": final_answer.to_string(), "mime_type": "image/png"},
        )
    elif isinstance(final_answer, AgentAudio):
        yield ChatMessage(
            role="assistant",
            content={"path": final_answer.to_string(), "mime_type": "audio/wav"},
        )
    else:


SUMMARY:
**Summary:**

The `stream_to_gradio` function runs an agent with the given task, streaming messages as Gradio chat logs. It handles different types of responses from the agent, including text, images, and audio.

**Key Implementation Details:**

* The function uses a try-except block to import `ChatMessage` from the `gradio` library.
* If in test mode, it defines a fictional `ChatMessage` class; otherwise, it raises an error if Grado is not installed.
* It iterates over the step logs of the agent's run, yielding messages using the `pull_message` function (not shown in this snippet).
* For different types of responses, it creates corresponding Gradio chat messages with specific content.

**Notable Dependencies and Requirements:**

* `gradio`: The library used for creating interactive demos.
* `AgentText`, `AgentImage`, and `AgentAudio`: Classes representing different agent response types (not shown in this snippet).


key: chroma:document
string_value:
CODE:

class Monitor:
    def __init__(self, tracked_llm_engine):
        self.step_durations = []
        self.tracked_llm_engine = tracked_llm_engine
        if getattr(self.tracked_llm_engine, "last_input_token_count", "Not found") != "Not found":
            self.total_input_token_count = 0
            self.total_output_token_count = 0

    def update_metrics(self, step_log):
        step_duration = step_log["step_duration"]
        self.step_durations.append(step_duration)
        logger.info(f"Step {len(self.step_durations)}:")
        logger.info(f"- Time taken: {step_duration:.2f} seconds (valid only if step succeeded)")

        if getattr(self.tracked_llm_engine, "last_input_token_count", None) is not None:
            self.total_input_token_count += self.tracked_llm_engine.last_input_token_count
            self.total_output_token_count += self.tracked_llm_engine.last_output_token_count
            logger.info(f"- Input tokens: {self.total_input_token_count}")


SUMMARY:
**Class Summary: Monitor**

The `Monitor` class tracks the performance of an LLM (Large Language Model) engine, providing metrics on step durations and token counts. The primary purpose is to collect data for analysis or logging.

**Key Implementation Details:**

* Calculates and stores step durations (`update_metrics` method)
* Updates cumulative token counts (`input_token_count` and `output_token_count`) using values from the `tracked_llm_engine`
* Logs performance metrics using a logger

**Notable Dependencies/Requirements:**

* ` tracked_llm_engine`: an object with attributes `last_input_token_count` and `last_output_token_count`

**Initialization (in `__init__` method):**

* Creates an empty list to store step durations (`step_durations`)
* Initializes `tracked_llm_engine` as an instance variable and sets up initial token counts based on its attributes.


key: chroma:document
string_value:
CODE:

class SpeechToTextTool(PipelineTool):
    default_checkpoint = "distil-whisper/distil-large-v3"
    description = "This is a tool that transcribes an audio into text. It returns the transcribed text."
    name = "transcriber"
    pre_processor_class = WhisperProcessor
    model_class = WhisperForConditionalGeneration

    inputs = {"audio": {"type": "audio", "description": "The audio to transcribe"}}
    output_type = "string"

    def encode(self, audio):
        return self.pre_processor(audio, return_tensors="pt")

    def forward(self, inputs):
        return self.model.generate(inputs["input_features"])

    def decode(self, outputs):


SUMMARY:
**Summary:**

`SpeechToTextTool` is a class that leverages the `WhisperProcessor` and `WhisperForConditionalGeneration` models to transcribe audio inputs into text.

**Key Functionality:**

- Transcribes audio inputs using `encode()` method
- Passes encoded input to `forward()` method for conditional generation
- Decodes output from `forward()` method using `decode()` method

**Implementation Details:**

- Uses `WhisperForConditionalGeneration` model for conditional generation
- Applies `skip_special_tokens=True` in `batch_decode()` method for decoding outputs
- Utilizes `return_tensors="pt"` when passing input to `pre_processor`

**Dependencies and Requirements:**

- `WhisperProcessor`
- `WhisperForConditionalGeneration`
- Hugging Face Transformers (implied by model checkpoint "distil-whisper/distil-large-v3")


key: chroma:document
string_value:
CODE:

class CustomFormatter(logging.Formatter):
    grey = "\x1b[38;20m"
    bold_yellow = "\x1b[33;1m"
    red = "\x1b[31;20m"
    green = "\x1b[32;20m"
    bold_green = "\x1b[32;20;1m"
    bold_red = "\x1b[31;1m"
    bold_white = "\x1b[37;1m"
    orange = "\x1b[38;5;214m"
    bold_orange = "\x1b[38;5;214;1m"
    reset = "\x1b[0m"
    format = "%(message)s"

    FORMATS = {
        logging.DEBUG: grey + format + reset,
        logging.INFO: format,
        logging.WARNING: bold_yellow + format + reset,
        logging.ERROR: red + format + reset,
        logging.CRITICAL: bold_red + format + reset,
        31: reset + format + reset,
        32: green + format + reset,
        33: bold_green + format + reset,
        34: bold_white + format + reset,
        35: orange + format + reset,
        36: bold_orange + format + reset,
    }

    def format(self, record):
        log_fmt = self.FORMATS.get(record.levelno)
        formatter = logging.Formatter(log_fmt)


SUMMARY:
**Summary**

The `CustomFormatter` class extends the built-in `logging.Formatter` to create a custom logger with colorized output. It defines ANSI escape codes for various color styles and uses them to format log messages according to their level (DEBUG, INFO, WARNING, ERROR, CRITICAL).

**Key Details**

*   Uses ANSI escape codes to apply different coloring to log messages based on their level or specific numbers.
*   Defines a `FORMATS` dictionary that maps logging levels to corresponding colorized formats.

**Notable Dependencies and Requirements**

None specified. The class only uses the built-in logging module, but inherits from it.

**Primary Functionality**
It provides a custom logger with colored output based on log level or specific numbers, allowing for more customizable logging in Python applications.


key: chroma:document
string_value:
CODE:

def parse_json_blob(json_blob: str) -> Dict[str, str]:
    try:
        first_accolade_index = json_blob.find("{")
        last_accolade_index = [a.start() for a in list(re.finditer("}", json_blob))][-1]
        json_blob = json_blob[first_accolade_index : last_accolade_index + 1].replace('\\"', "'")
        json_data = json.loads(json_blob, strict=False)
        return json_data
    except json.JSONDecodeError as e:
        place = e.pos
        if json_blob[place - 1 : place + 2] == "},\n":
            raise ValueError(
                "JSON is invalid: you probably tried to provide multiple tool calls in one action. PROVIDE ONLY ONE TOOL CALL."
            )
        raise ValueError(
            f"The JSON blob you used is invalid due to the following error: {e}.\n"
            f"JSON blob was: {json_blob}, decoding failed on that specific part of the blob:\n"
            f"'{json_blob[place-4:place+5]}'."
        )
    except Exception as e:


SUMMARY:
**Summary**

The `parse_json_blob` function parses a JSON-formatted string, extracting and decoding the internal JSON data. It checks for validation errors by analyzing the extracted JSON structure.

**Main Purpose and Functionality**: Extracts JSON data from an input string while preserving its formatting (quotes with double backslashes become single quotes).

**Key Implementation Details**:

*   Uses `json.loads` to parse the extracted JSON data.
*   Checks for JSON decoding errors, handling specific cases:
    *   If the error occurs after a `\x7D,`.
    *   Provides more detailed error messages when possible.

**Notable Dependencies and Requirements**:

*   The `json` module (specifically `json.loads`, `json.JSONDecodeError`) is required for parsing JSON data.
*   Python 2 support due to use of `strict=False` in `json.loads`.
*


key: chroma:document
string_value:
CODE:

def parse_code_blob(code_blob: str) -> str:
    try:
        pattern = r"```(?:py|python)?\n(.*?)\n```"
        match = re.search(pattern, code_blob, re.DOTALL)
        return match.group(1).strip()
    except Exception as e:
        raise ValueError(
            f"""
The code blob you used is invalid: due to the following error: {e}
This means that the regex pattern {pattern} was not respected: make sure to include code with the correct pattern, for instance:
Thoughts: Your thoughts
Code:
```py
# Your python code here
```<end_action>"""


SUMMARY:
**Summary**

`parse_code_blob` function takes a string `code_blob` as input and returns the inner code snippet surrounded by triple backticks (`). It uses a regular expression pattern to extract the code. If the input is invalid, it raises a ValueError with an error message that includes the original exception.

**Key Implementation Details**

* Uses regex pattern: `r"```(?:py|python)?\n(.*?)\n```"``
	+ Pattern extracts inner code snippet (matched by `(.*?)`): `{match.group(1).strip()}`,
* Includes error handling:
	+ Catches all exceptions and raises a ValueError with error message including the original exception.
	+ Provides an example of correct input format for thoughts and code.

**Notable Dependencies or Requirements**

* Python's `re` module is used for regular expression matching.
* Input string must be in the format: " Thoughts \n Code: ```py ...```" where "...".


key: chroma:document
string_value:
CODE:

def parse_json_tool_call(json_blob: str) -> Tuple[str, Dict[str, str]]:
    json_blob = json_blob.replace("```json", "").replace("```", "")
    tool_call = parse_json_blob(json_blob)
    if "action" in tool_call and "action_input" in tool_call:
        return tool_call["action"], tool_call["action_input"]
    elif "action" in tool_call:
        return tool_call["action"], None
    else:
        raise ValueError(
            f"Missing keys: {[key for key in ['action', 'action_input'] if key not in tool_call]} in blob {tool_call}"


SUMMARY:
**Summary**

The `parse_json_tool_call` function parses a JSON-formatted string, typically output from the `json_tool` command. It:

* Removes surrounding JSON comments (````json`) and extracts the parsed JSON data using `parse_json_blob`.
* Checks for presence of required keys (`action` and/or `action_input`). If both present, returns both values; if only `action` is present, returns `action` with `None` as the second value. Raises a `ValueError` if either key is missing.
* Returns type hint: `Tuple[str, Dict[str, str]]`, indicating it returns a string and a dictionary.

**Key Implementation Details**

* Uses `json_blob.replace()` to remove comments before parsing (````json```).
* Utilizes `parse_json_blob` (implementation not shown) to extract JSON data.
* Verifies presence of required keys using boolean expressions.

**Notable Dependencies or Requirements**

* No external dependencies mentioned, but likely assumes the existence of the `parse_json_blob` function.


key: chroma:document
string_value:
CODE:

def parse_text_tool_call(text: str) -> Tuple[str, Union[str, Dict[str, str]]]:
    """
    Expects a text in the format: 'Action:', 'Action input:', 'Observation:'. 'Action input:' contains a json string with input arguments.
    """
    try:
        if "Observation:" in text:
            text = text.split("Observation:")[0]
        if "Action:" in text:
            text = text.split("Action:")[1]
        tool_name, tool_input = text.split("Action input:")
        if "{" in tool_input:
            tool_input = parse_json_blob(tool_input)
        else:
            tool_input = tool_input.strip().replace('"', "")
        return tool_name.strip().replace('"', "").replace("\\", ""), tool_input
    except Exception as e:
        raise ValueError(
            f"Error in parsing the text tool call: {e}. Be sure to provide the correct format. DO NOT repeat your previous incorrect tool call."


SUMMARY:
**Function Summary**

`parse_text_tool_call` takes a string input `text` and parses its contents to extract `tool_name` and `tool_input`.

**Primary Purpose**
The primary purpose of this function is to parse the input text in a specific format, extract the action name and optional JSON- formatted input arguments, and return them as a tuple.

**Key Implementation Details**

*   The function splits the input text into three parts based on the presence of 'Action:', 'Observation:', and 'Action input:'.
*   It then splits 'tool_input' further if it starts with '{', indicating a JSON blob, and processes it accordingly.
*   If 'tool_input' contains non-JSON data (i.e., no '{'), it is parsed as plain text by stripping whitespace and replacing double quotes and backslashes.

**Notable Dependencies or Requirements**

None explicitly mentioned. However, the `parse_json_blob` function is not defined in this snippet, suggesting it might be a separate module or defined elsewhere in the codebase.

**Return Value**
A tuple containing `tool_name` (as a string) and either a `str`, no quotes ("", without backslashes), `dict` of strings for tool_input or JSON- formatted input arguments.


key: chroma:document
string_value:
CODE:

def to_text(input: Union[List[Dict[str, str]], Dict[str, str], str]) -> str:
    if isinstance(input, list):
        return "\n".join([m["content"] for m in input])
    elif isinstance(input, dict):
        return input["content"]
    else:


SUMMARY:
**Summary:**

The `to_text` function converts its input into a string representation. It takes three possible inputs:

- A list of dictionaries (`List[Dict[str, str]]`) where the resulting text is joined with newline characters (`\n`).
- A dictionary (`Dict[str, str]`) where only one key ("content") is extracted.
- A string (`str`) which remains unchanged.

**Implementation Details:**

* Uses ` Union[List[Dict[str, str]], Dict[str, str], str]` to handle different input formats.
* Utilizes list comprehension and conditional statements for concise processing.

**Notable Dependencies:**

* None

**Interfaces/Methods:**

* Does not implement any external interfaces or methods, uses built-in Python functionality.


key: chroma:document
string_value:
CODE:

class Toolbox:
    """
    The toolbox contains all tools that the agent can perform operations with, as well as a few methods to
    manage them.

    Args:
        tools (`List[Tool]`):
            The list of tools to instantiate the toolbox with
        add_base_tools (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to add the tools available within `transformers` to the toolbox.
    """

    def __init__(self, tools: List[Tool], add_base_tools: bool = False):
        self._tools = {tool.name: tool for tool in tools}
        if add_base_tools:
            self.add_base_tools()
        self._load_tools_if_needed()

    def add_base_tools(self, add_python_interpreter: bool = False):
        global _tools_are_initialized
        global HUGGINGFACE_DEFAULT_TOOLS
        if not _tools_are_initialized:
            HUGGINGFACE_DEFAULT_TOOLS = setup_default_tools(logger)
            _tools_are_initialized = True
        for tool in HUGGINGFACE_DEFAULT_TOOLS.values():
            if tool.name != "python_interpreter" or add_python_interpreter:
                self.add_tool(tool)
        self._load_tools_if_needed()

    @property
    def tools(self) -> Dict[str, Tool]:
        """Get all tools currently in the toolbox"""
        return self._tools

    def show_tool_descriptions(self, tool_description_template: str = None) -> str:
        """
        Returns the description of all tools in the toolbox

        Args:
            tool_description_template (`str`, *optional*):
                The template to use to describe the tools. If not provided, the default template will be used.
        """
        return "\n".join(
            [get_tool_description_with_args(tool, tool_description_template) for tool in self._tools.values()]
        )

    def add_tool(self, tool: Tool):
        """
        Adds a tool to the toolbox

        Args:
            tool (`Tool`):
                The tool to add to the toolbox.
        """
        if tool.name in self._tools:
            raise KeyError(f"Error: tool '{tool.name}' already exists in the toolbox.")
        self._tools[tool.name] = tool

    def remove_tool(self, tool_name: str):
        """
        Removes a tool from the toolbox

        Args:
            tool_name (`str`):
                The tool to remove from the toolbox.
        """
        if tool_name not in self._tools:
            raise KeyError(
                f"Error: tool {tool_name} not found in toolbox for removal, should be instead one of {list(self._tools.keys())}."
            )
        del self._tools[tool_name]

    def update_tool(self, tool: Tool):
        """
        Updates a tool in the toolbox according to its name.

        Args:
            tool (`Tool`):
                The tool to update to the toolbox.
        """
        if tool.name not in self._tools:
            raise KeyError(
                f"Error: tool {tool.name} not found in toolbox for update, should be instead one of {list(self._tools.keys())}."
            )
        self._tools[tool.name] = tool

    def clear_toolbox(self):
        """Clears the toolbox"""
        self._tools = {}

    def _load_tools_if_needed(self):
        for name, tool in self._tools.items():
            if not isinstance(tool, Tool):
                task_or_repo_id = tool.task if tool.repo_id is None else tool.repo_id
                self._tools[name] = load_tool(task_or_repo_id)

    def __repr__(self):
        toolbox_description = "Toolbox contents:\n"
        for tool in self._tools.values():
            toolbox_description += f"\t{tool.name}: {tool.description}\n"


SUMMARY:
**Summary: Toolbox Class**

The `Toolbox` class is a centralized container for managing tools that an agent can perform operations with. It provides methods to add, remove, update, and clear tools.

**Main Purpose and Functionality**

*   Manages a collection of tools (`List[Tool]`) and their metadata.
*   Provides methods for adding, removing, updating, and clearing tools in the toolbox.

**Key Implementation Details**

*   The `add_base_tools` method initializes the default tools provided by the `transformers` library.
*   The `_load_tools_if_needed` method loads tools if they have not been initialized.
*   The `show_tool_descriptions` method returns a string containing descriptions of all tools in the toolbox.

**Important Methods and Interfaces**

*   `add_tool(Tool)`: Adds a tool to the toolbox.
*   `remove_tool(str)`: Removes a tool from the toolbox by its name.
*   `update_tool(Tool)`: Updates a tool in the toolbox according to its name.
*   `clear_toolbox()`: Clears all tools from the toolbox.

**Notable Dependencies and Requirements**

*   The `transformers` library provides default tools for initialization ( `_tools_are_initialized` and `setup_default_tools(logger) `).
*   External dependencies are required for loading tools (`load_tool(task_or_repo_id)`).

**Classes/Functions Mentioned:**

*   `Tool`
*   `Logger`

Note: This summary is a condensed version of the original code chunk, focusing on key aspects and terminology.


key: chroma:document
string_value:
CODE:

class AgentError(Exception):
    """Base class for other agent-related exceptions"""

    def __init__(self, message):
        super().__init__(message)


SUMMARY:
**Class Summary:**

`AgentError`: A base exception class for agent-related errors.

**Purpose and Functionality:**
Raises an error with a given message, serving as the foundation for more specific agent-specific exceptions.

**Key Implementation Details:**
- Initializes the exception with a `message` parameter using `super().__init__(message)`.
- Stores the provided `message` locally to facilitate further information retrieval.

**Notable Dependencies/Requirements:**

* Relies on Python's built-in `Exception` class.
* No external dependencies listed.


key: chroma:document
string_value:
CODE:

class AgentParsingError(AgentError):
    """Exception raised for errors in parsing in the agent"""



SUMMARY:
**Summary**

A custom exception class `AgentParsingError` is defined, inheriting from `AgentError`. This class represents an error that occurs during parsing of agency-related data.

**Key Details**

* **Purpose**: Raises an exception for parsing errors in agent-related data
* **Implementation Details**
	+ No specific implementation or methods are provided; it serves as a base exception class.
* **Notable Dependencies/Requirements**: Inherited from `AgentError` (exact name referenced).


key: chroma:document
string_value:
CODE:

class AgentExecutionError(AgentError):
    """Exception raised for errors in execution in the agent"""



SUMMARY:
**Class Summary:**

`AgentExecutionError`

* **Purpose**: Represents exceptions related to execution errors in an `Agent`.

**Key Implementation Details:**
- This is a custom exception class inheriting from `AgentError`.
- No additional implementation or behavior is defined.

**Notable Dependencies:**
- Inherits from `AgentError`.


key: chroma:document
string_value:
CODE:

class AgentMaxIterationsError(AgentError):
    """Exception raised for errors in execution in the agent"""



SUMMARY:
**Summary:**

`AgentMaxIterationsError` is a custom exception class that inherits from `AgentError`. Its primary purpose is to raise an error when the maximum number of iterations exceeds, indicating an issue with the agent's execution. 

**Key Implementation Details:**
 None specified in this code chunk.

**Notable Dependencies or Requirements:**
- Inherited from `AgentError`


key: chroma:document
string_value:
CODE:

class AgentGenerationError(AgentError):
    """Exception raised for errors in generation in the agent"""



SUMMARY:
Here is a concise and focused summary:

**Summary:** `AgentGenerationError` Exception
=============================================

*   **Purpose**: Rare exception raised due to errors in agent generation.
*   **Key Implementation**: None (an empty class, inheriting from `AgentError`)
*   **Notable Dependency**: None (`AgentError` is not explicitly imported beyond inheritance)
*   **Interfaces**: Inherits from `AgentError`

This summary adheres to the provided guidelines, focusing on key information in a concise manner while excluding unnecessary details.


key: chroma:document
string_value:
CODE:

def format_prompt_with_tools(toolbox: Toolbox, prompt_template: str, tool_description_template: str) -> str:
    tool_descriptions = toolbox.show_tool_descriptions(tool_description_template)
    prompt = prompt_template.replace("<<tool_descriptions>>", tool_descriptions)

    if "<<tool_names>>" in prompt:
        tool_names = [f"'{tool_name}'" for tool_name in toolbox.tools.keys()]
        prompt = prompt.replace("<<tool_names>>", ", ".join(tool_names))



SUMMARY:
**Function Summary**

`format_prompt_with_tools`

Formats a template prompt using the provided `toolbox`, `prompt_template`, and `tool_description_template`. The function:

* Replaces placeholders with tool descriptions from the `toolbox`.
* Expands a list of tool names within the prompt.

**Key Details:**

* Input:
	+ `toolbox`: An instance of `Toolbox` containing available tools.
	+ `prompt_template`: A string template for the formatted prompt.
	+ `tool_description_template`: A string template for tool descriptions.
* Output:
	+ The formatted prompt with replaced placeholders and expanded tool names.

**Implementation:**

The function employs list comprehensions, string replacements, and basic string formatting. It relies on an instance of `Toolbox` providing access to its tools and their descriptions.

**Interfaces/Dependencies:**

* None explicit; relies solely on the `Toolbox` class provided by the toolbox.


key: chroma:document
string_value:
CODE:

def show_agents_descriptions(managed_agents: list):
    managed_agents_descriptions = """
You can also give requests to team members.
Calling a team member works the same as for calling a tool: simply, the only argument you can give in the call is 'request', a long string explaning your request.
Given that this team member is a real human, you should be very verbose in your request.
Here is a list of the team members that you can call:"""
    for agent in managed_agents.values():
        managed_agents_descriptions += f"\n- {agent.name}: {agent.description}"


SUMMARY:
**Function Summary:**

`show_agents_descriptions` displays descriptions of team members and instructions on requesting tasks from them.

**Key Details:**

* Takes a `managed_agents` dictionary as input, which contains agent objects with `name` and `description` attributes.
* Iterates over the agents and appends their descriptions to a formatted string.
* Returns the complete text describing team members and task request instructions.

**Notable Dependencies/Interfaces:** None mentioned in this chunk.


key: chroma:document
string_value:
CODE:

def format_prompt_with_managed_agents_descriptions(prompt_template, managed_agents=None) -> str:
    if managed_agents is not None:
        return prompt_template.replace("<<managed_agents_descriptions>>", show_agents_descriptions(managed_agents))
    else:


SUMMARY:
**Function Summary**

`format_prompt_with_managed_agents_descriptions`

**Purpose**: Formats a prompt template by replacing the `<<managed_agents_descriptions>>` placeholder with the formatted descriptions of managed agents.

**Implementation Details**:

* Uses `replace()` to replace the placeholder, conditionally using either the actual agent descriptions or an empty string.
* References `show_agents_descriptions(managed_agents)` for formatting the agent descriptions, but its implementation is not shown in this code chunk.

**Dependencies/Requirements**:

* None explicitly mentioned, but it assumes the existence of a `show_agents_descriptions` function.


key: chroma:document
string_value:
CODE:

def format_prompt_with_imports(prompt_template: str, authorized_imports: List[str]) -> str:
    if "<<authorized_imports>>" not in prompt_template:
        raise AgentError("Tag '<<authorized_imports>>' should be provided in the prompt.")


SUMMARY:
**Summary:**

Formats a prompt template by inserting authorized imports.

**Key Details:**

* Function `format_prompt_with_imports` takes:
	+ `prompt_template`: The base prompt string with a placeholder for authorized imports (`<<authorized_imports>>`)
	+ `authorized_imports`: A list of authorized import statements as strings
* Replaces the placeholder in `prompt_template` with the formatted authorized imports.
* Raises an `AgentError` if the necessary tag is missing from the prompt template.

**Notable Dependencies:**

None

**Implementational Notes:**

* Uses string replacement to insert authorized imports into the prompt template.


key: chroma:document
string_value:
CODE:

class Agent:
    def __init__(
        self,
        tools: Union[List[Tool], Toolbox],
        llm_engine: Callable = None,
        system_prompt: Optional[str] = None,
        tool_description_template: Optional[str] = None,
        additional_args: Dict = {},
        max_iterations: int = 6,
        tool_parser: Optional[Callable] = None,
        add_base_tools: bool = False,
        verbose: int = 0,
        grammar: Optional[Dict[str, str]] = None,
        managed_agents: Optional[List] = None,
        step_callbacks: Optional[List[Callable]] = None,
        monitor_metrics: bool = True,
    ):
        if system_prompt is None:
            system_prompt = DEFAULT_REACT_CODE_SYSTEM_PROMPT
        if tool_parser is None:
            tool_parser = parse_json_tool_call
        self.agent_name = self.__class__.__name__
        self.llm_engine = llm_engine
        self.system_prompt_template = system_prompt
        self.tool_description_template = (
            tool_description_template if tool_description_template else DEFAULT_TOOL_DESCRIPTION_TEMPLATE
        )
        self.additional_args = additional_args
        self.max_iterations = max_iterations
        self.logger = logger
        self.tool_parser = tool_parser
        self.grammar = grammar

        self.managed_agents = None
        if managed_agents is not None:
            self.managed_agents = {agent.name: agent for agent in managed_agents}

        if isinstance(tools, Toolbox):
            self._toolbox = tools
            if add_base_tools:
                if not is_torch_available():
                    raise ImportError("Using the base tools requires torch to be installed.")

                self._toolbox.add_base_tools(add_python_interpreter=(self.__class__ == ReactJsonAgent))
        else:
            self._toolbox = Toolbox(tools, add_base_tools=add_base_tools)
        self._toolbox.add_tool(FinalAnswerTool())

        self.system_prompt = format_prompt_with_tools(
            self._toolbox, self.system_prompt_template, self.tool_description_template
        )
        self.system_prompt = format_prompt_with_managed_agents_descriptions(self.system_prompt, self.managed_agents)
        self.prompt = None
        self.logs = []
        self.task = None

        if verbose == 0:
            logger.setLevel(logging.WARNING)
        elif verbose == 1:
            logger.setLevel(logging.INFO)
        elif verbose == 2:
            logger.setLevel(logging.DEBUG)

        # Initialize step callbacks
        self.step_callbacks = step_callbacks if step_callbacks is not None else []

        # Initialize Monitor if monitor_metrics is True
        self.monitor = None
        if monitor_metrics:
            self.monitor = Monitor(self.llm_engine)
            self.step_callbacks.append(self.monitor.update_metrics)

    @property
    def toolbox(self) -> Toolbox:
        """Get the toolbox currently available to the agent"""
        return self._toolbox

    def initialize_for_run(self):
        self.token_count = 0
        self.system_prompt = format_prompt_with_tools(
            self._toolbox,
            self.system_prompt_template,
            self.tool_description_template,
        )
        self.system_prompt = format_prompt_with_managed_agents_descriptions(self.system_prompt, self.managed_agents)
        if hasattr(self, "authorized_imports"):
            self.system_prompt = format_prompt_with_imports(
                self.system_prompt, list(set(LIST_SAFE_MODULES) | set(self.authorized_imports))
            )
        self.logs = [{"system_prompt": self.system_prompt, "task": self.task}]
        self.logger.log(33, "======== New task ========")
        self.logger.log(34, self.task)
        self.logger.debug("System prompt is as follows:")
        self.logger.debug(self.system_prompt)

    def write_inner_memory_from_logs(self, summary_mode: Optional[bool] = False) -> List[Dict[str, str]]:
        """
        Reads past llm_outputs, actions, and observations or errors from the logs into a series of messages
        that can be used as input to the LLM.
        """
        prompt_message = {"role": MessageRole.SYSTEM, "content": self.logs[0]["system_prompt"]}
        task_message = {
            "role": MessageRole.USER,
            "content": "Task: " + self.logs[0]["task"],
        }
        if summary_mode:
            memory = [task_message]
        else:
            memory = [prompt_message, task_message]
        for i, step_log in enumerate(self.logs[1:]):
            if "llm_output" in step_log and not summary_mode:
                thought_message = {"role": MessageRole.ASSISTANT, "content": step_log["llm_output"].strip()}
                memory.append(thought_message)
            if "facts" in step_log:
                thought_message = {
                    "role": MessageRole.ASSISTANT,
                    "content": "[FACTS LIST]:\n" + step_log["facts"].strip(),
                }
                memory.append(thought_message)

            if "plan" in step_log and not summary_mode:
                thought_message = {"role": MessageRole.ASSISTANT, "content": "[PLAN]:\n" + step_log["plan"].strip()}
                memory.append(thought_message)

            if "tool_call" in step_log and summary_mode:
                tool_call_message = {
                    "role": MessageRole.ASSISTANT,
                    "content": f"[STEP {i} TOOL CALL]: " + str(step_log["tool_call"]).strip(),
                }
                memory.append(tool_call_message)

            if "task" in step_log:
                tool_call_message = {
                    "role": MessageRole.USER,
                    "content": "New task:\n" + step_log["task"],
                }
                memory.append(tool_call_message)

            if "error" in step_log or "observation" in step_log:
                if "error" in step_log:
                    message_content = (
                        f"[OUTPUT OF STEP {i}] -> Error:\n"
                        + str(step_log["error"])
                        + "\nNow let's retry: take care not to repeat previous errors! If you have retried several times, try a completely different approach.\n"
                    )
                elif "observation" in step_log:
                    message_content = f"[OUTPUT OF STEP {i}] -> Observation:\n{step_log['observation']}"
                tool_response_message = {"role": MessageRole.TOOL_RESPONSE, "content": message_content}
                memory.append(tool_response_message)

        return memory

    def get_succinct_logs(self):
        return [{key: value for key, value in log.items() if key != "agent_memory"} for log in self.logs]

    def extract_action(self, llm_output: str, split_token: str) -> str:
        """
        Parse action from the LLM output

        Args:
            llm_output (`str`): Output of the LLM
            split_token (`str`): Separator for the action. Should match the example in the system prompt.
        """
        try:
            split = llm_output.split(split_token)
            rationale, action = (
                split[-2],
                split[-1],
            )  # NOTE: using indexes starting from the end solves for when you have more than one split_token in the output
        except Exception as e:
            self.logger.error(e, exc_info=1)
            raise AgentParsingError(
                f"Error: No '{split_token}' token provided in your output.\nYour output:\n{llm_output}\n. Be sure to include an action, prefaced with '{split_token}'!"
            )
        return rationale.strip(), action.strip()

    def execute_tool_call(self, tool_name: str, arguments: Dict[str, str]) -> Any:
        """
        Execute tool with the provided input and returns the result.
        This method replaces arguments with the actual values from the state if they refer to state variables.

        Args:
            tool_name (`str`): Name of the Tool to execute (should be one from self.toolbox).
            arguments (Dict[str, str]): Arguments passed to the Tool.
        """
        available_tools = self.toolbox.tools
        if self.managed_agents is not None:
            available_tools = {**available_tools, **self.managed_agents}
        if tool_name not in available_tools:
            error_msg = f"Error: unknown tool {tool_name}, should be instead one of {list(available_tools.keys())}."
            self.logger.error(error_msg, exc_info=1)
            raise AgentExecutionError(error_msg)

        try:
            if isinstance(arguments, str):
                observation = available_tools[tool_name](arguments)
            elif isinstance(arguments, dict):
                for key, value in arguments.items():
                    # if the value is the name of a state variable like "image.png", replace it with the actual value
                    if isinstance(value, str) and value in self.state:
                        arguments[key] = self.state[value]
                observation = available_tools[tool_name](**arguments)
            else:
                raise AgentExecutionError(
                    f"Arguments passed to tool should be a dict or string: got a {type(arguments)}."
                )
            return observation
        except Exception as e:
            if tool_name in self.toolbox.tools:
                raise AgentExecutionError(
                    f"Error in tool call execution: {e}\nYou should only use this tool with a correct input.\n"
                    f"As a reminder, this tool's description is the following:\n{get_tool_description_with_args(available_tools[tool_name])}"
                )
            elif tool_name in self.managed_agents:
                raise AgentExecutionError(
                    f"Error in calling team member: {e}\nYou should only ask this team member with a correct request.\n"
                    f"As a reminder, this team member's description is the following:\n{available_tools[tool_name]}"
                )

    def log_rationale_code_action(self, rationale: str, code_action: str) -> None:
        self.logger.warning("=== Agent thoughts:")
        self.logger.log(31, rationale)
        self.logger.warning(">>> Agent is executing the code below:")
        if is_pygments_available():
            self.logger.log(
                31, highlight(code_action, PythonLexer(ensurenl=False), Terminal256Formatter(style="nord"))
            )
        else:
            self.logger.log(31, code_action)
        self.logger.warning("====")

    def run(self, **kwargs):
        """To be implemented in the child class"""


SUMMARY:
This is a Python class that appears to be part of an AI or machine learning framework. The class inherits from another class (not shown) and implements some specific functionality related to executing tools, extracting actions from LLM (Large Language Model) outputs, and logging agent thoughts.

Here are some key aspects of the code:

1. **Toolbox**: The `toolbox` attribute holds a dictionary of available tools that can be executed. Each tool is associated with a function definition in this class.
2. **Action extraction**: The `extract_action` method takes an LLM output and a split token as input and attempts to extract the reasoning and action from the output. If an error occurs, it raises an exception with a descriptive message.
3. **Tool execution**: The `execute_tool_call` method executes a tool with the given arguments and returns the result. It checks if the tool exists in the toolbox and tries to execute it. If an error occurs during execution, it raises an exception.
4. **Rationale logging**: The `log_rationale_code_action` method logs the agent's thoughts (rationale) and code actions. It uses a pygments-aware logger to nicely format the code snippet.
5. **LLM output handling**: Some parts of the class seem to handle the LLM output in different ways, such as splitting it into words or checking if the output is a string.

This code appears to be a part of a library that provides an interface for agents to interact with tools and execute tasks. It might be used in various domains like AI, robotics, games, or other applications where complex thinking and problem-solving are required.

Some potential improvements could include:

1. **Error handling**: While the class raises informative exceptions when errors occur, it might be beneficial to add more specific error types for different cases.
2. **Code organization**: The code seems to be a mix of logging, tool execution, and agent logic. It might be helpful to separate these concerns into different methods or classes for better readability.
3. **Type hinting**: Adding type hints could make the code easier to understand and use, especially when it comes to arguments and return types.

Here's an example of how this class could be reorganized with more descriptive docstrings:

```python
class Agent:
    def __init__(self):
        """
        Initialize the agent.

        Attributes:
            toolbox (dict): Dictionary of available tools.
        """
        self.toolbox = {}

    def execute_tool(self, tool_name, arguments):
        """
        Execute a tool with given arguments.

        Args:
            tool_name (str): Name of the tool to execute.
            arguments (dict or str): Arguments for the tool.

        Returns:
            Any: Result of the tool execution.
        """
        if tool_name not in self.toolbox:
            raise AgentExecutionError(f"Unknown tool {tool_name}")
        
        # ... implementation ...

    def extract_action(self, llm_output, split_token):
        """
        Extract action from LLM output.

        Args:
            llm_output (str): Output of the Large Language Model.
            split_token (str): Separator for the action.

        Returns:
            str: Reasoning and action from the output.
        """
        try:
            # ... implementation ...
        except Exception as e:
            raise AgentParsingError(f"Error extracting action: {e}")

    def log_rationale_code_action(self, rationale, code_action):
        """
        Log agent thoughts (rationale) and code actions.

        Args:
            rationale (str): Reasoning thought.
            code_action (str): Code snippet to be executed.
        """
        self.logger.warning("=== Agent thoughts:")
        self.logger.log(31, rationale)
        self.logger.warning(">>> Agent is executing the code below:")
        if is_pygments_available():
            # ... format the code snippet ...
        else:
            self.logger.log(31, code_action)
```

This reorganization makes it easier to understand what each method does and how they are connected. It also leaves room for potential improvements or additions.


key: chroma:document
string_value:
CODE:

class CodeAgent(Agent):
    """
    A class for an agent that solves the given task using a single block of code. It plans all its actions, then executes all in one shot.
    """

    def __init__(
        self,
        tools: List[Tool],
        llm_engine: Optional[Callable] = None,
        system_prompt: Optional[str] = None,
        tool_description_template: Optional[str] = None,
        grammar: Optional[Dict[str, str]] = None,
        additional_authorized_imports: Optional[List[str]] = None,
        **kwargs,
    ):
        if llm_engine is None:
            llm_engine = HfApiEngine()
        if system_prompt is None:
            system_prompt = DEFAULT_CODE_SYSTEM_PROMPT
        if tool_description_template is None:
            tool_description_template = DEFAULT_TOOL_DESCRIPTION_TEMPLATE
        super().__init__(
            tools=tools,
            llm_engine=llm_engine,
            system_prompt=system_prompt,
            tool_description_template=tool_description_template,
            grammar=grammar,
            **kwargs,
        )

        if not is_pygments_available():
            transformers_logging.warning_once(
                logger,
                "pygments isn't installed. Installing pygments will enable color syntax highlighting in the "
                "CodeAgent.",
            )

        self.python_evaluator = evaluate_python_code
        self.additional_authorized_imports = additional_authorized_imports if additional_authorized_imports else []
        self.authorized_imports = list(set(LIST_SAFE_MODULES) | set(self.additional_authorized_imports))
        self.system_prompt = self.system_prompt.replace("<<authorized_imports>>", str(self.authorized_imports))

    def parse_code_blob(self, result: str) -> str:
        """
        Override this method if you want to change the way the code is
        cleaned in the `run` method.
        """
        return parse_code_blob(result)

    def run(self, task: str, return_generated_code: bool = False, **kwargs):
        """
        Runs the agent for the given task.

        Args:
            task (`str`): The task to perform
            return_generated_code (`bool`, *optional*, defaults to `False`): Whether to return the generated code instead of running it
            kwargs (additional keyword arguments, *optional*):
                Any keyword argument to send to the agent when evaluating the code.

        Example:

        ```py
        from transformers.agents import CodeAgent

        agent = CodeAgent(tools=[])
        agent.run("What is the result of 2 power 3.7384?")
        ```
        """
        self.task = task
        if len(kwargs) > 0:
            self.task += f"\nYou have been provided with these initial arguments: {str(kwargs)}."
        self.state = kwargs.copy()
        self.initialize_for_run()

        # Run LLM
        prompt_message = {"role": MessageRole.SYSTEM, "content": self.system_prompt}
        task_message = {
            "role": MessageRole.USER,
            "content": "Task: " + self.task,
        }

        self.prompt = [prompt_message, task_message]
        self.logger.info("====Executing with this prompt====")
        self.logger.info(self.prompt)

        additional_args = {"grammar": self.grammar} if self.grammar is not None else {}
        llm_output = self.llm_engine(self.prompt, stop_sequences=["<end_action>"], **additional_args)

        if return_generated_code:
            return llm_output

        # Parse
        try:
            rationale, code_action = self.extract_action(llm_output=llm_output, split_token="Code:")
        except Exception as e:
            self.logger.debug(
                f"Error in extracting action, trying to parse the whole output as code. Error trace: {e}"
            )
            rationale, code_action = "", llm_output

        try:
            code_action = self.parse_code_blob(code_action)
        except Exception as e:
            error_msg = f"Error in code parsing: {e}. Be sure to provide correct code"
            self.logger.error(error_msg, exc_info=1)
            return error_msg

        # Execute
        self.log_rationale_code_action(rationale, code_action)
        try:
            available_tools = {**BASE_PYTHON_TOOLS.copy(), **self.toolbox.tools}
            output = self.python_evaluator(
                code_action,
                static_tools=available_tools,
                custom_tools={},
                state=self.state,
                authorized_imports=self.authorized_imports,
            )
            self.logger.info(self.state["print_outputs"])
            return output
        except Exception as e:
            error_msg = f"Error in execution: {e}. Be sure to provide correct code."
            self.logger.error(error_msg, exc_info=1)


SUMMARY:
**Class Summary**

`CodeAgent` is an agent class that solves tasks using a single block of code. It plans all actions and executes them in one shot.

*   **Main Purpose**: Plans and executes code for a given task using LLM (Large Language Model).
*   **Key Implementation Details**:
    *   Utilizes `HfApiEngine` to run LLM.
    *   Processes input task, generates response from LLM, parses result, executes code in Python evaluator.
    *   Uses `parse_code_blob` to process parsed code.
    *   Runs code in `python_evaluator` with available tools and state.
*   **Notable Dependencies**:
    +   `HfApiEngine`
    +   `Transformer.logging`

**Example Usage**
```python
from transformers.agents import CodeAgent

agent = CodeAgent(tools=[])
agent.run("What is the result of 2 power 3.7384?")
```
This summary captures the core functionality and implementation details of the `CodeAgent` class, making it easier for developers to understand and work with this code component in their RAG system.


key: chroma:document
string_value:
CODE:

class ReactAgent(Agent):
    """
    This agent that solves the given task step by step, using the ReAct framework:
    While the objective is not reached, the agent will perform a cycle of thinking and acting.
    The action will be parsed from the LLM output: it consists in calls to tools from the toolbox, with arguments chosen by the LLM engine.
    """

    def __init__(
        self,
        tools: List[Tool],
        llm_engine: Optional[Callable] = None,
        system_prompt: Optional[str] = None,
        tool_description_template: Optional[str] = None,
        grammar: Optional[Dict[str, str]] = None,
        plan_type: Optional[str] = None,
        planning_interval: Optional[int] = None,
        **kwargs,
    ):
        if llm_engine is None:
            llm_engine = HfApiEngine()
        if system_prompt is None:
            system_prompt = DEFAULT_REACT_CODE_SYSTEM_PROMPT
        if tool_description_template is None:
            tool_description_template = DEFAULT_TOOL_DESCRIPTION_TEMPLATE
        if plan_type is None:
            plan_type = SUPPORTED_PLAN_TYPES[0]
        else:
            assert plan_type in SUPPORTED_PLAN_TYPES, f"plan type {plan_type} is not supported"
        super().__init__(
            tools=tools,
            llm_engine=llm_engine,
            system_prompt=system_prompt,
            tool_description_template=tool_description_template,
            grammar=grammar,
            **kwargs,
        )
        self.planning_interval = planning_interval
        self.plan_type = plan_type

    def provide_final_answer(self, task) -> str:
        """
        This method provides a final answer to the task, based on the logs of the agent's interactions.
        """
        self.prompt = [
            {
                "role": MessageRole.SYSTEM,
                "content": "An agent tried to answer an user query but it got stuck and failed to do so. You are tasked with providing an answer instead. Here is the agent's memory:",
            }
        ]
        self.prompt += self.write_inner_memory_from_logs()[1:]
        self.prompt += [
            {
                "role": MessageRole.USER,
                "content": f"Based on the above, please provide an answer to the following user request:\n{task}",
            }
        ]
        try:
            return self.llm_engine(self.prompt)
        except Exception as e:
            return f"Error in generating final llm output: {e}."

    def run(self, task: str, stream: bool = False, reset: bool = True, **kwargs):
        """
        Runs the agent for the given task.

        Args:
            task (`str`): The task to perform

        Example:
        ```py
        from transformers.agents import ReactCodeAgent
        agent = ReactCodeAgent(tools=[])
        agent.run("What is the result of 2 power 3.7384?")
        ```
        """
        self.task = task
        if len(kwargs) > 0:
            self.task += f"\nYou have been provided with these initial arguments: {str(kwargs)}."
        self.state = kwargs.copy()
        if reset:
            self.initialize_for_run()
        else:
            self.logs.append({"task": task})
        if stream:
            return self.stream_run(task)
        else:
            return self.direct_run(task)

    def stream_run(self, task: str):
        """
        Runs the agent in streaming mode, yielding steps as they are executed: should be launched only in the `run` method.
        """
        final_answer = None
        iteration = 0
        while final_answer is None and iteration < self.max_iterations:
            step_start_time = time.time()
            step_log_entry = {"iteration": iteration, "start_time": step_start_time}
            try:
                self.step(step_log_entry)
                if "final_answer" in step_log_entry:
                    final_answer = step_log_entry["final_answer"]
            except AgentError as e:
                self.logger.error(e, exc_info=1)
                step_log_entry["error"] = e
            finally:
                step_end_time = time.time()
                step_log_entry["step_end_time"] = step_end_time
                step_log_entry["step_duration"] = step_end_time - step_start_time
                self.logs.append(step_log_entry)
                for callback in self.step_callbacks:
                    callback(step_log_entry)
                iteration += 1
                yield step_log_entry

        if final_answer is None and iteration == self.max_iterations:
            error_message = "Reached max iterations."
            final_step_log = {"error": AgentMaxIterationsError(error_message)}
            self.logs.append(final_step_log)
            self.logger.error(error_message, exc_info=1)
            final_answer = self.provide_final_answer(task)
            final_step_log["final_answer"] = final_answer
            final_step_log["step_duration"] = 0
            for callback in self.step_callbacks:
                callback(final_step_log)
            yield final_step_log

        yield final_answer

    def direct_run(self, task: str):
        """
        Runs the agent in direct mode, returning outputs only at the end: should be launched only in the `run` method.
        """
        final_answer = None
        iteration = 0
        while final_answer is None and iteration < self.max_iterations:
            step_start_time = time.time()
            step_log_entry = {"iteration": iteration, "start_time": step_start_time}
            try:
                if self.planning_interval is not None and iteration % self.planning_interval == 0:
                    self.planning_step(task, is_first_step=(iteration == 0), iteration=iteration)
                self.step(step_log_entry)
                if "final_answer" in step_log_entry:
                    final_answer = step_log_entry["final_answer"]
            except AgentError as e:
                self.logger.error(e, exc_info=1)
                step_log_entry["error"] = e
            finally:
                step_end_time = time.time()
                step_log_entry["step_end_time"] = step_end_time
                step_log_entry["step_duration"] = step_end_time - step_start_time
                self.logs.append(step_log_entry)
                for callback in self.step_callbacks:
                    callback(step_log_entry)
                iteration += 1

        if final_answer is None and iteration == self.max_iterations:
            error_message = "Reached max iterations."
            final_step_log = {"error": AgentMaxIterationsError(error_message)}
            self.logs.append(final_step_log)
            self.logger.error(error_message, exc_info=1)
            final_answer = self.provide_final_answer(task)
            final_step_log["final_answer"] = final_answer
            final_step_log["step_duration"] = 0
            for callback in self.step_callbacks:
                callback(final_step_log)

        return final_answer

    def planning_step(self, task, is_first_step: bool = False, iteration: int = None):
        """
        Used periodically by the agent to plan the next steps to reach the objective.

        Args:
            task (`str`): The task to perform
            is_first_step (`bool`): If this step is not the first one, the plan should be an update over a previous plan.
            iteration (`int`): The number of the current step, used as an indication for the LLM.
        """
        if is_first_step:
            message_prompt_facts = {"role": MessageRole.SYSTEM, "content": SYSTEM_PROMPT_FACTS}
            message_prompt_task = {
                "role": MessageRole.USER,
                "content": f"""Here is the task:
```
{task}
```
Now begin!""",
            }

            answer_facts = self.llm_engine([message_prompt_facts, message_prompt_task])

            message_system_prompt_plan = {
                "role": MessageRole.SYSTEM,
                "content": PROMPTS_FOR_INITIAL_PLAN[self.plan_type]["system"],
            }
            message_user_prompt_plan = {
                "role": MessageRole.USER,
                "content": PROMPTS_FOR_INITIAL_PLAN[self.plan_type]["user"].format(
                    task=task,
                    tool_descriptions=self._toolbox.show_tool_descriptions(self.tool_description_template),
                    managed_agents_descriptions=(
                        show_agents_descriptions(self.managed_agents) if self.managed_agents is not None else ""
                    ),
                    answer_facts=answer_facts,
                ),
            }
            answer_plan = self.llm_engine(
                [message_system_prompt_plan, message_user_prompt_plan], stop_sequences=["<end_plan>"]
            )

            final_plan_redaction = f"""Here is the plan of action that I will follow to solve the task:
```
{answer_plan}
```"""
            final_facts_redaction = f"""Here are the facts that I know so far:
```
{answer_facts}
```""".strip()
            self.logs.append({"plan": final_plan_redaction, "facts": final_facts_redaction})
            self.logger.log(36, "===== Initial plan =====")
            self.logger.log(35, final_plan_redaction)
        else:  # update plan
            agent_memory = self.write_inner_memory_from_logs(
                summary_mode=False
            )  # This will not log the plan but will log facts

            # Redact updated facts
            facts_update_system_prompt = {
                "role": MessageRole.SYSTEM,
                "content": SYSTEM_PROMPT_FACTS_UPDATE,
            }
            facts_update_message = {
                "role": MessageRole.USER,
                "content": USER_PROMPT_FACTS_UPDATE,
            }
            facts_update = self.llm_engine([facts_update_system_prompt] + agent_memory + [facts_update_message])

            # Redact updated plan
            plan_update_message = {
                "role": MessageRole.SYSTEM,
                "content": PROMPTS_FOR_PLAN_UPDATE[self.plan_type]["system"].format(task=task),
            }
            plan_update_message_user = {
                "role": MessageRole.USER,
                "content": PROMPTS_FOR_PLAN_UPDATE[self.plan_type]["user"].format(
                    task=task,
                    tool_descriptions=self._toolbox.show_tool_descriptions(self.tool_description_template),
                    managed_agents_descriptions=(
                        show_agents_descriptions(self.managed_agents) if self.managed_agents is not None else ""
                    ),
                    facts_update=facts_update,
                    remaining_steps=(self.max_iterations - iteration),
                ),
            }
            plan_update = self.llm_engine(
                [plan_update_message] + agent_memory + [plan_update_message_user], stop_sequences=["<end_plan>"]
            )

            # Log final facts and plan
            final_plan_redaction = PLAN_UPDATE_FINAL_PLAN_REDACTION.format(task=task, plan_update=plan_update)
            final_facts_redaction = f"""Here is the updated list of the facts that I know:
```
{facts_update}
```"""
            self.logs.append({"plan": final_plan_redaction, "facts": final_facts_redaction})
            self.logger.log(36, "===== Updated plan =====")


SUMMARY:
The provided text appears to be a class implementation in Python for an AI system that utilizes reinforcement learning and natural language processing (NLP) techniques. Here's a concise summary of the code:

**Class Overview**

This class is designed to handle tasks, plans, and decisions made by an AI agent. It uses reinforcement learning (RL) to optimize its behavior.

**Key Classes and Methods**

1. **Agent**: The main class that inherits from an undefined base class or interface.
	* `planning_step`: Periodically updates the plan based on new information collected about the environment.
	* `provide_plan_update` returns additional facts to update the plan.
2. **RL Agent**: This is where reinforcement learning algorithms are implemented, such as policy gradient and Q-learning.
3. **Utility Functions**:
	* `prepare_initial_plan`: Generates initial prompts for planning based on the task's description and available tools.
	* `prepare_answer`: Provides responses to questions about tasks and agents' abilities.

**Core Concepts**

1. Reinforcement Learning (RL): The AI agent learns by interacting with its environment, receiving rewards or penalties for actions taken.
2. Planning: Periodically updates the plan to optimize behavior based on new information collected about the environment.
3. Natural Language Processing (NLP): The AI uses NLP techniques to understand and generate human language.

**Notes**

* This implementation lacks explicit context regarding the task domain, problem scope, or specific algorithm architecture used.
* There are many commented lines and variables; some may be worth removing or refactoring to improve code readability.

**Missing Information**

To provide more comprehensive feedback, I would need more information about:

1. The specific reinforcement learning algorithms implemented (e.g., policy gradient methods, Q-learning).
2. The role of additional data structures like a planning queue or a dictionary for storing plan-related variables.
3. Any additional libraries or techniques used beyond NLP and RL (e.g., graph optimization, matrix operations).

**Recommendations**

To improve this codebase:

1. Refactor variable names to be more descriptive.
2. Consider using more informative naming conventions for methods and attributes.
3. Improve the structure and organization of comments and docstrings to make it easier to follow code flow.
4. Investigate potential improvements in performance or accuracy, such as exploring different hyperparameters for RL algorithms.

Keep up the good work on developing a comprehensive AI system!


key: chroma:document
string_value:
CODE:

class ReactJsonAgent(ReactAgent):
    """
    This agent that solves the given task step by step, using the ReAct framework:
    While the objective is not reached, the agent will perform a cycle of thinking and acting.
    The tool calls will be formulated by the LLM in JSON format, then parsed and executed.
    """

    def __init__(
        self,
        tools: List[Tool],
        llm_engine: Optional[Callable] = None,
        system_prompt: Optional[str] = None,
        tool_description_template: Optional[str] = None,
        grammar: Optional[Dict[str, str]] = None,
        planning_interval: Optional[int] = None,
        **kwargs,
    ):
        if llm_engine is None:
            llm_engine = HfApiEngine()
        if system_prompt is None:
            system_prompt = DEFAULT_REACT_JSON_SYSTEM_PROMPT
        if tool_description_template is None:
            tool_description_template = DEFAULT_TOOL_DESCRIPTION_TEMPLATE
        super().__init__(
            tools=tools,
            llm_engine=llm_engine,
            system_prompt=system_prompt,
            tool_description_template=tool_description_template,
            grammar=grammar,
            planning_interval=planning_interval,
            **kwargs,
        )

    def step(self, log_entry: Dict[str, Any]):
        """
        Perform one step in the ReAct framework: the agent thinks, acts, and observes the result.
        The errors are raised here, they are caught and logged in the run() method.
        """
        agent_memory = self.write_inner_memory_from_logs()

        self.prompt = agent_memory
        self.logger.debug("===== New step =====")

        # Add new step in logs
        log_entry["agent_memory"] = agent_memory.copy()

        self.logger.info("===== Calling LLM with this last message: =====")
        self.logger.info(self.prompt[-1])

        try:
            additional_args = {"grammar": self.grammar} if self.grammar is not None else {}
            llm_output = self.llm_engine(
                self.prompt, stop_sequences=["<end_action>", "Observation:"], **additional_args
            )
        except Exception as e:
            raise AgentGenerationError(f"Error in generating llm output: {e}.")
        self.logger.debug("===== Output message of the LLM: =====")
        self.logger.debug(llm_output)
        log_entry["llm_output"] = llm_output

        # Parse
        self.logger.debug("===== Extracting action =====")
        rationale, action = self.extract_action(llm_output=llm_output, split_token="Action:")

        try:
            tool_name, arguments = self.tool_parser(action)
        except Exception as e:
            raise AgentParsingError(f"Could not parse the given action: {e}.")

        log_entry["rationale"] = rationale
        log_entry["tool_call"] = {"tool_name": tool_name, "tool_arguments": arguments}

        # Execute
        self.logger.warning("=== Agent thoughts:")
        self.logger.log(31, rationale)
        self.logger.warning(f">>> Calling tool: '{tool_name}' with arguments: {arguments}")
        if tool_name == "final_answer":
            if isinstance(arguments, dict):
                if "answer" in arguments:
                    answer = arguments["answer"]
                    if (
                        isinstance(answer, str) and answer in self.state.keys()
                    ):  # if the answer is a state variable, return the value
                        answer = self.state[answer]
                else:
                    answer = arguments
            else:
                answer = arguments
            log_entry["final_answer"] = answer
            return answer
        else:
            if arguments is None:
                arguments = {}
            observation = self.execute_tool_call(tool_name, arguments)
            observation_type = type(observation)
            if observation_type in [AgentImage, AgentAudio]:
                if observation_type == AgentImage:
                    observation_name = "image.png"
                elif observation_type == AgentAudio:
                    observation_name = "audio.mp3"
                # TODO: observation naming could allow for different names of same type

                self.state[observation_name] = observation
                updated_information = f"Stored '{observation_name}' in memory."
            else:
                updated_information = str(observation).strip()
            self.logger.info(updated_information)
            log_entry["observation"] = updated_information


SUMMARY:
**Summary**

```markdown
# ReactJsonAgent Class

The `ReactJsonAgent` class implements the ReAct framework for solving given tasks step by step. It takes in various parameters, initializes an instance of its parent class (`ReactAgent`), and performs a cycle of thinking and acting.

## Key Functionality

* Initializes and executes tasks using LLM (LLaMA) formulation
* Parses tool calls from JSON format
* Executes tools with the parsed actions

## Notable Dependencies

* `HfApiEngine`
* `DEFAULT_REACT_JSON_SYSTEM_prompt` template
* `DEFAULT_TOOL_DESCRIPTION_TEMPLATE`

## Key Implementation Details**

* The agent performs a cycle of thinking and acting, using LLM to formulate tool calls in JSON format.
* It extracts actions from the LLM output and parses them into tools with arguments.
* It executes these tools and updates its state accordingly.

## Exceptions**

Raises:
- `AgentGenerationError`
- `AgentParsingError`

Returns:
- Final answer
- Observation update
```

Please note that the summary focuses on the main purpose, key implementation details, and notable dependencies of the given code chunk.


key: chroma:document
string_value:
CODE:

class ReactCodeAgent(ReactAgent):
    """
    This agent that solves the given task step by step, using the ReAct framework:
    While the objective is not reached, the agent will perform a cycle of thinking and acting.
    The tool calls will be formulated by the LLM in code format, then parsed and executed.
    """

    def __init__(
        self,
        tools: List[Tool],
        llm_engine: Optional[Callable] = None,
        system_prompt: Optional[str] = None,
        tool_description_template: Optional[str] = None,
        grammar: Optional[Dict[str, str]] = None,
        additional_authorized_imports: Optional[List[str]] = None,
        planning_interval: Optional[int] = None,
        **kwargs,
    ):
        if llm_engine is None:
            llm_engine = HfApiEngine()
        if system_prompt is None:
            system_prompt = DEFAULT_REACT_CODE_SYSTEM_PROMPT
        if tool_description_template is None:
            tool_description_template = DEFAULT_TOOL_DESCRIPTION_TEMPLATE
        super().__init__(
            tools=tools,
            llm_engine=llm_engine,
            system_prompt=system_prompt,
            tool_description_template=tool_description_template,
            grammar=grammar,
            planning_interval=planning_interval,
            **kwargs,
        )

        if not is_pygments_available():
            transformers_logging.warning_once(
                logger,
                "pygments isn't installed. Installing pygments will enable color syntax highlighting in the "
                "ReactCodeAgent.",
            )

        self.python_evaluator = evaluate_python_code
        self.additional_authorized_imports = additional_authorized_imports if additional_authorized_imports else []
        self.authorized_imports = list(set(LIST_SAFE_MODULES) | set(self.additional_authorized_imports))
        self.system_prompt = self.system_prompt.replace("<<authorized_imports>>", str(self.authorized_imports))
        self.custom_tools = {}

    def step(self, log_entry: Dict[str, Any]):
        """
        Perform one step in the ReAct framework: the agent thinks, acts, and observes the result.
        The errors are raised here, they are caught and logged in the run() method.
        """
        agent_memory = self.write_inner_memory_from_logs()

        self.prompt = agent_memory.copy()
        self.logger.debug("===== New step =====")

        # Add new step in logs
        log_entry["agent_memory"] = agent_memory.copy()

        self.logger.info("===== Calling LLM with these last messages: =====")
        self.logger.info(self.prompt[-2:])

        try:
            additional_args = {"grammar": self.grammar} if self.grammar is not None else {}
            llm_output = self.llm_engine(
                self.prompt, stop_sequences=["<end_action>", "Observation:"], **additional_args
            )
        except Exception as e:
            raise AgentGenerationError(f"Error in generating llm output: {e}.")

        self.logger.debug("=== Output message of the LLM:")
        self.logger.debug(llm_output)
        log_entry["llm_output"] = llm_output

        # Parse
        self.logger.debug("=== Extracting action ===")
        try:
            rationale, raw_code_action = self.extract_action(llm_output=llm_output, split_token="Code:")
        except Exception as e:
            self.logger.debug(f"Error in extracting action, trying to parse the whole output. Error trace: {e}")
            rationale, raw_code_action = llm_output, llm_output

        try:
            code_action = parse_code_blob(raw_code_action)
        except Exception as e:
            error_msg = f"Error in code parsing: {e}. Make sure to provide correct code"
            raise AgentParsingError(error_msg)

        log_entry["rationale"] = rationale
        log_entry["tool_call"] = {"tool_name": "code interpreter", "tool_arguments": code_action}

        # Execute
        self.log_rationale_code_action(rationale, code_action)
        try:
            static_tools = {
                **BASE_PYTHON_TOOLS.copy(),
                **self.toolbox.tools,
            }
            if self.managed_agents is not None:
                static_tools = {**static_tools, **self.managed_agents}
            result = self.python_evaluator(
                code_action,
                static_tools=static_tools,
                custom_tools=self.custom_tools,
                state=self.state,
                authorized_imports=self.authorized_imports,
            )
            self.logger.warning("Print outputs:")
            self.logger.log(32, self.state["print_outputs"])
            observation = "Print outputs:\n" + self.state["print_outputs"]
            if result is not None:
                self.logger.warning("Last output from code snippet:")
                self.logger.log(32, str(result))
                observation += "Last output from code snippet:\n" + str(result)[:100000]
            log_entry["observation"] = observation
        except Exception as e:
            error_msg = f"Code execution failed due to the following error:\n{str(e)}"
            if "'dict' object has no attribute 'read'" in str(e):
                error_msg += "\nYou get this error because you passed a dict as input for one of the arguments instead of a string."
            raise AgentExecutionError(error_msg)
        for line in code_action.split("\n"):
            if line[: len("final_answer")] == "final_answer":
                self.logger.log(33, "Final answer:")
                self.logger.log(32, result)
                log_entry["final_answer"] = result


SUMMARY:
**ReactCodeAgent Class Summary**

The `ReactCodeAgent` class is a ReAct framework agent that solves tasks step by step. It utilizes an LLM (Large Language Model) engine to generate and parse code.

**Class Functionality:**

*   Initializes the agent with tools, LLM engine, system prompt, and additional authorized imports.
*   Performs a cycle of thinking and acting on steps until the objective is reached.
*   Calls the LLM tool with predefined arguments and parses the output to extract action and code execution results.

**Key Implementation Details:**

*   Uses `evaluate_python_code` as the Python evaluator for code execution.
*   Utilizes `pygments` for color syntax highlighting if available.
*   Includes a step method for performing one step in the ReAct framework.

**Dependencies and Requirements:**

*   `HfApiEngine`: An LLM engine for generating and parsing output.
*   `Pygments`: A library for color syntax highlighting (if installed).
*   `Tool`: An interface for tools used by the agent.
*   `BASE_PYTHON_TOOLS`, `toolbox`, and `managed_agents` are external dependencies that implement static tools, toolboxes, and managed agents, respectively.

**Notable Method Signatures:**

*   `__init__`: Initializes the ReactCodeAgent with various parameters and super initializes the parent class.
*   `step`: Performs a step in the ReAct framework using the LLM engine to generate output.

This summary provides an overview of the main purpose and functionality, key implementation details, and dependencies/requirements for the `ReactCodeAgent` class.


key: chroma:document
string_value:
CODE:

class ManagedAgent:
    def __init__(self, agent, name, description, additional_prompting=None, provide_run_summary=False):
        self.agent = agent
        self.name = name
        self.description = description
        self.additional_prompting = additional_prompting
        self.provide_run_summary = provide_run_summary

    def write_full_task(self, task):
        full_task = f"""You're a helpful agent named '{self.name}'.
You have been submitted this task by your manager.
---
Task:
{task}
---
You're helping your manager solve a wider task: so make sure to not provide a one-line answer, but give as much information as possible so that they have a clear understanding of the answer.

Your final_answer WILL HAVE to contain these parts:
### 1. Task outcome (short version):
### 2. Task outcome (extremely detailed version):
### 3. Additional context (if relevant):

Put all these in your final_answer tool, everything that you do not pass as an argument to final_answer will be lost.
And even if your task resolution is not successful, please return as much context as possible, so that your manager can act upon this feedback.
<<additional_prompting>>"""
        if self.additional_prompting:
            full_task = full_task.replace("\n<<additional_prompting>>", self.additional_prompting).strip()
        else:
            full_task = full_task.replace("\n<<additional_prompting>>", "").strip()
        return full_task

    def __call__(self, request, **kwargs):
        full_task = self.write_full_task(request)
        output = self.agent.run(full_task, **kwargs)
        if self.provide_run_summary:
            answer = f"Here is the final answer from your managed agent '{self.name}':\n"
            answer += str(output)
            answer += f"\n\nFor more detail, find below a summary of this agent's work:\nSUMMARY OF WORK FROM AGENT '{self.name}':\n"
            for message in self.agent.write_inner_memory_from_logs(summary_mode=True):
                content = message["content"]
                if len(str(content)) < LENGTH_TRUNCATE_REPORTS or "[FACTS LIST]" in str(content):
                    answer += "\n" + str(content) + "\n---"
                else:
                    answer += (
                        "\n"
                        + str(content)[:LENGTH_TRUNCATE_REPORTS]
                        + "\n(...Step was truncated because too long)...\n---"
                    )
            answer += f"\nEND OF SUMMARY OF WORK FROM AGENT '{self.name}'."
            return answer
        else:


SUMMARY:
**Class Summary: ManagedAgent**

The `ManagedAgent` class is designed to manage and interact with an agent, generating a detailed prompt for task execution while providing additional context. Its primary purpose is to:

*   Initialize the managed agent with an agent, name, description, and optional parameters for added prompting.
*   Generate a full task prompt containing instructions, task information, and context requirements.
*   Run the agent's task prompt and retrieve output.

**Key Implementation Details:**

*   The `write_full_task` method generates a full task prompt by formatting instructions, task information, and context requirements into a string.
*   The `__call__` method runs the managed agent's task prompt, executes the task with the provided parameters, and returns the output or generates a run summary if requested.

**Notable Dependencies and Requirements:**

*   The class uses an instance of `agent`, which is assumed to have its own implementation for running tasks (`run` method).
*   It requires the `LENGTH_TRUNCATE_REPORTS` constant defined elsewhere in the codebase.
*   The `write_inner_memory_from_logs` method from the `agent` instance is used to generate a summary of the agent's work.


key: chroma:document
string_value:
CODE:

class MessageRole(str, Enum):
    USER = "user"
    ASSISTANT = "assistant"
    SYSTEM = "system"
    TOOL_CALL = "tool-call"
    TOOL_RESPONSE = "tool-response"

    @classmethod
    def roles(cls):


SUMMARY:
**Class Summary: MessageRole**

The `MessageRole` class defines an enumeration of message role types. It provides a list of pre-defined roles with corresponding string values, utilizing the built-in Python `Enum`. The primary purpose is to categorize and identify different roles in message exchanges.

**Key Implementation Details**

*   `roles()` class method: returns a list of all predefined role strings.
*   Enum values are assigned using class constants (`USER`, etc.), which preserve exact names for easy reference.

**Notable Dependencies or Requirements**

*   Built-in Python `Enum`
*   No external dependencies.


key: chroma:document
string_value:
CODE:

def get_clean_message_list(message_list: List[Dict[str, str]], role_conversions: Dict[str, str] = {}):
    """
    Subsequent messages with the same role will be concatenated to a single message.

    Args:
        message_list (`List[Dict[str, str]]`): List of chat messages.
    """
    final_message_list = []
    message_list = deepcopy(message_list)  # Avoid modifying the original list
    for message in message_list:
        if not set(message.keys()) == {"role", "content"}:
            raise ValueError("Message should contain only 'role' and 'content' keys!")

        role = message["role"]
        if role not in MessageRole.roles():
            raise ValueError(f"Incorrect role {role}, only {MessageRole.roles()} are supported for now.")

        if role in role_conversions:
            message["role"] = role_conversions[role]

        if len(final_message_list) > 0 and message["role"] == final_message_list[-1]["role"]:
            final_message_list[-1]["content"] += "\n=======\n" + message["content"]
        else:
            final_message_list.append(message)


SUMMARY:
**Function Summary: `get_clean_message_list`**

**Purpose:** Concatenates messages with the same role in a chat message list.

**Implementation Details:**

* Filters out invalid message keys and unsupported roles.
* Applies custom role conversions if available.
* Merges subsequent messages with the same role into a single entry, separated by a separator string "\n=======\n".

**Key Signatures:**
```python
from typing import List, Dict

def get_clean_message_list(
    message_list: List[Dict[str, str]],
    role_conversions: Dict[str, str] = {}
) -> List[Dict[str, str]]:
```

**Notable Dependencies:**

* `MessageRole.roles()` (returns a set of supported roles)
* `deepcopy` (for creating a copy of the original message list)

**Interfaces:**
- `List[Dict[str, str]]`: Input message list
- `Dict[str, str]`: Single message in the list


key: chroma:document
string_value:
CODE:

class HfEngine:
    def __init__(self, model_id: Optional[str] = None):
        self.last_input_token_count = None
        self.last_output_token_count = None
        if model_id is None:
            model_id = "HuggingFaceTB/SmolLM2-1.7B-Instruct"
            logger.warning(f"Using default model for token counting: '{model_id}'")
        try:
            self.tokenizer = AutoTokenizer.from_pretrained(model_id)
        except Exception as e:
            logger.warning(f"Failed to load tokenizer for model {model_id}: {e}. Loading default tokenizer instead.")
            self.tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM2-1.7B-Instruct")

    def get_token_counts(self):
        return {
            "input_token_count": self.last_input_token_count,
            "output_token_count": self.last_output_token_count,
        }

    def generate(
        self, messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None
    ):
        raise NotImplementedError

    def __call__(
        self, messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None
    ) -> str:
        """Process the input messages and return the model's response.

        This method sends a list of messages to the Hugging Face Inference API, optionally with stop sequences and grammar customization.

        Parameters:
            messages (`List[Dict[str, str]]`):
                A list of message dictionaries to be processed. Each dictionary should have the structure `{"role": "user/system", "content": "message content"}`.
            stop_sequences (`List[str]`, *optional*):
                A list of strings that will stop the generation if encountered in the model's output.
            grammar (`str`, *optional*):
                The grammar or formatting structure to use in the model's response.

        Returns:
            `str`: The text content of the model's response.

        Example:
            ```python
            >>> engine = HfApiEngine(
            ...     model="meta-llama/Meta-Llama-3.1-8B-Instruct",
            ...     token="your_hf_token_here",
            ...     max_tokens=2000
            ... )
            >>> messages = [{"role": "user", "content": "Explain quantum mechanics in simple terms."}]
            >>> response = engine(messages, stop_sequences=["END"])
            >>> print(response)
            "Quantum mechanics is the branch of physics that studies..."
            ```
        """
        if not isinstance(messages, List):
            raise ValueError("Messages should be a list of dictionaries with 'role' and 'content' keys.")
        if stop_sequences is None:
            stop_sequences = []
        response = self.generate(messages, stop_sequences, grammar)
        self.last_input_token_count = len(self.tokenizer.apply_chat_template(messages, tokenize=True))
        self.last_output_token_count = len(self.tokenizer.encode(response))

        # Remove stop sequences from LLM output
        for stop_seq in stop_sequences:
            if response[-len(stop_seq) :] == stop_seq:
                response = response[: -len(stop_seq)]


SUMMARY:
**Summary: HfEngine Class**

The `HfEngine` class is a client for the Hugging Face Inference API, used to generate human-like responses based on user input. The primary purpose is to process messages through the model's API.

**Implementation Details:**

1.  Initializes with an optional `model_id`, defaulting to "HuggingFaceTB/SmolLM2-1.7B-Instruct".
2.  Loads a tokenizer for the provided or default `model_id` using `AutoTokenizer.from_pretrained`.
3.  The `generate` method is overridden to raise a NotImplementedError, requiring implementation in child classes.
4.  The `__call__` (method call) method processes input messages and returns the model's response.

**Key Method Signatures:**

1.  `self.generate(messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None)`
2.  `self.__call__(messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None) -> str`

**Notable Dependencies and Requirements:**

1.  `AutoTokenizer` from Hugging Face
2.  The "HugingFaceTB/SmolLM2-1.7B-Instruct" model (optional)

**Class Inheritance:**

No explicit inheritance is provided. However, it inherits interfaces and functions from the parent class (`BaseEngine`) or other external libraries.


key: chroma:document
string_value:
CODE:

class HfApiEngine(HfEngine):
    """A class to interact with Hugging Face's Inference API for language model interaction.

    This engine allows you to communicate with Hugging Face's models using the Inference API. It can be used in both serverless mode or with a dedicated endpoint, supporting features like stop sequences and grammar customization.

    Parameters:
        model (`str`, *optional*, defaults to `"meta-llama/Meta-Llama-3.1-8B-Instruct"`):
            The Hugging Face model ID to be used for inference. This can be a path or model identifier from the Hugging Face model hub.
        token (`str`, *optional*):
            Token used by the Hugging Face API for authentication.
            If not provided, the class will use the token stored in the Hugging Face CLI configuration.
        max_tokens (`int`, *optional*, defaults to 1500):
            The maximum number of tokens allowed in the output.
        timeout (`int`, *optional*, defaults to 120):
            Timeout for the API request, in seconds.

    Raises:
        ValueError:
            If the model name is not provided.
    """

    def __init__(
        self,
        model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct",
        token: Optional[str] = None,
        max_tokens: Optional[int] = 1500,
        timeout: Optional[int] = 120,
    ):
        super().__init__(model_id=model)
        self.model = model
        self.client = InferenceClient(self.model, token=token, timeout=timeout)
        self.max_tokens = max_tokens

    def generate(
        self, messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None
    ) -> str:
        # Get clean message list
        messages = get_clean_message_list(messages, role_conversions=llama_role_conversions)

        # Send messages to the Hugging Face Inference API
        if grammar is not None:
            response = self.client.chat_completion(
                messages, stop=stop_sequences, max_tokens=self.max_tokens, response_format=grammar
            )
        else:
            response = self.client.chat_completion(messages, stop=stop_sequences, max_tokens=self.max_tokens)

        response = response.choices[0].message.content


SUMMARY:
**Hugging Face Inference API Engine Class Summary**

### Purpose and Functionality

The `HfApiEngine` class interacts with Hugging Face's Inference API for language model interaction. It allows serverless mode or dedicated endpoint usage, supporting features like stop sequences and grammar customization.

### Key Implementation Details

* Initializes with a specified Hugging Face model ID (`model`), token (`token`), maximum tokens (`max_tokens`), and timeout (`timeout`)
* Includes an `InferenceClient` instance for API requests
* Employs the `chat_completion` method to send messages to the Hugging Face Inference API, generating responses based on specified grammar or stop sequences

### Notable Dependencies and Requirements

* `HuggingFace` ( InferenceClient ) model ID (`model`)
* Authentication token (`token`)
* Maximum number of tokens allowed (`max_tokens`)
* Timeout for API request (`timeout`)
* `get_clean_message_list`
* `llama_role_conversions`

### Method Signatures and Interfaces

* `generate(messages: List[Dict[str, str]], stop_sequences: Optional[List[str]] = None, grammar: Optional[str] = None) -> str`


key: chroma:document
string_value:
CODE:

class TransformersEngine(HfEngine):
    """This engine uses a pre-initialized local text-generation pipeline."""

    def __init__(self, pipeline: Pipeline, model_id: Optional[str] = None):
        super().__init__(model_id)
        self.pipeline = pipeline

    def generate(
        self,
        messages: List[Dict[str, str]],
        stop_sequences: Optional[List[str]] = None,
        grammar: Optional[str] = None,
        max_length: int = 1500,
    ) -> str:
        # Get clean message list
        messages = get_clean_message_list(messages, role_conversions=llama_role_conversions)

        # Get LLM output
        if stop_sequences is not None and len(stop_sequences) > 0:
            stop_strings = stop_sequences
        else:
            stop_strings = None

        output = self.pipeline(
            messages,
            stop_strings=stop_strings,
            max_length=max_length,
            tokenizer=self.pipeline.tokenizer,
        )

        response = output[0]["generated_text"][-1]["content"]


SUMMARY:
**Transformer Engine Class Summary**

```markdown
# Purpose and Functionality
The `TransformersEngine` class is an extension of the `HfEngine` class, utilizing a pre-initialized local text-generation pipeline to generate responses.

# Key Implementation Details
- Uses the `pipeline` object to process input messages (`generate` method)
- Applies stop sequences if provided, or uses no stop strings by default

## Class Variables and Inheritance

* Inherit from `HfEngine`
* Use a `pipeline` object (Instantiated with optional `model_id`)
```

**Method Signature Breakdown**

### `generate(methodSignature)`

* Input: `messages`, `stop_sequences`, `grammar`, and `max_length` parameters.
* Returns the generated response as a string (`str`).


key: chroma:document
string_value:
CODE:

def download_prompt(prompt_or_repo_id, agent_name, mode="run"):
    """
    Downloads and caches the prompt from a repo and returns it contents (if necessary).
    """
    if prompt_or_repo_id is None:
        prompt_or_repo_id = DEFAULT_PROMPTS_REPO

    # prompt is considered a repo ID when it does not contain any kind of space
    if re.search("\\s", prompt_or_repo_id) is not None:
        return prompt_or_repo_id

    prompt_file = cached_file(
        prompt_or_repo_id, PROMPT_FILES[mode], repo_type="dataset", user_agent={"agent": agent_name}
    )
    with open(prompt_file, "r", encoding="utf-8") as f:


SUMMARY:
**Function Summary: `download_prompt`**

**Primary Purpose:** Downloads and caches a prompt from a repository or returns an existing cached result.

**Main Functionality:**

1. **Prompt Validation and Return**: Checks if `prompt_or_repo_id` contains whitespace, returning it as is if true.
2. **Cache Request**: Requests the cached file for the specified `prompt_file` using `cached_file`.
3. **File Reading**: Reads the contents of the requested `prompt_file` using Python's `open` function.

**Key Implementation Details:**

* Use of `re.search` for whitespace checking
* Utilization of `cached_file` and open functions

**Dependencies/Requirements:**

* `DEFAULT_PROMPTS_REPO`
* `PROMPT_FILES` dictionary with mode-dependent configuration
* External dependency: `cached_file` function

**Interface/Method Signatures:**

- `download_prompt(prompt_or_repo_id, agent_name, mode="run")`

Note: The default value for the `mode` parameter is `"run"`, which implies that a different `PROMPT_FILES` dictionary might be used in other scenarios.


key: chroma:document
string_value:
CODE:

class TranslationTool(PipelineTool):
    """
    Example:

    ```py
    from transformers.agents import TranslationTool

    translator = TranslationTool()
    translator("This is a super nice API!", src_lang="English", tgt_lang="French")
    ```
    """

    lang_to_code = LANGUAGE_CODES
    default_checkpoint = "facebook/nllb-200-distilled-600M"
    description = (
        "This is a tool that translates text from a language to another."
        f"Both `src_lang`and `tgt_lang` should belong to this list of languages: {list(lang_to_code.keys())}."
    )
    name = "translator"
    pre_processor_class = AutoTokenizer
    model_class = AutoModelForSeq2SeqLM

    inputs = {
        "text": {"type": "string", "description": "The text to translate"},
        "src_lang": {
            "type": "string",
            "description": "The language of the text to translate. Written in plain English, such as 'Romanian', or 'Albanian'",
        },
        "tgt_lang": {
            "type": "string",
            "description": "The language for the desired ouput language. Written in plain English, such as 'Romanian', or 'Albanian'",
        },
    }
    output_type = "string"

    def encode(self, text, src_lang, tgt_lang):
        if src_lang not in self.lang_to_code:
            raise ValueError(f"{src_lang} is not a supported language.")
        if tgt_lang not in self.lang_to_code:
            raise ValueError(f"{tgt_lang} is not a supported language.")
        src_lang = self.lang_to_code[src_lang]
        tgt_lang = self.lang_to_code[tgt_lang]
        return self.pre_processor._build_translation_inputs(
            text, return_tensors="pt", src_lang=src_lang, tgt_lang=tgt_lang
        )

    def forward(self, inputs):
        return self.model.generate(**inputs)

    def decode(self, outputs):


SUMMARY:
**Summary:**

Class `TranslationTool` is a PyTorch pipeline for text translation. 

*   **Purpose:** Translate text from one language to another using Seq2Seq LM model (`AutoModelForSeq2SeqLM`) with `AutoTokenizer` preprocessing.

***Key Implementation Details:***

*   `encode`: Preprocesses input text for translation.
    *   Checks if source and target languages are supported and converts them into the code representation (`LANGUAGE_CODES`).
*   `forward`: Runs the model's generation process on preprocessed inputs to produce translated output.
*   `decode`: Post-processes translation outputs using `post_processor.decode`.

***Notable Dependencies:***

*   `transformers.agents` for pipelines and models
*   `pytorch`
*   `HuggingFace` for AutoTokenizer and AutoModelForSeq2SeqLM

***Interface/Class Hierarchy:***

*   Implements `PipelineTool` (not shown)
*   Inherits from ` PipelineTool`: uses a hierarchical interface


key: chroma:document
string_value:
CODE:

class DuckDuckGoSearchTool(Tool):
    name = "web_search"
    description = """Perform a web search based on your query (think a Google search) then returns the top search results as a list of dict elements.
    Each result has keys 'title', 'href' and 'body'."""
    inputs = {"query": {"type": "string", "description": "The search query to perform."}}
    output_type = "any"

    def forward(self, query: str) -> str:
        try:
            from duckduckgo_search import DDGS
        except ImportError:
            raise ImportError(
                "You must install package `duckduckgo_search` to run this tool: for instance run `pip install duckduckgo-search`."
            )
        results = DDGS().text(query, max_results=7)


SUMMARY:
**Summary**

The `DuckDuckGoSearchTool` class performs a web search based on the provided query and returns the top 7 search results as a list of dictionaries with keys 'title', 'href', and 'body'. 

**Class Attributes:**

*   `name`: "web_search"
*   `description`: Provides a human-readable description of the tool's functionality.
*   `inputs`: Specifies the input parameters required by the tool, in this case, a string query.
*   `output_type`: Defined as "any," indicating that the output can be of any data type.

**Methods:**

*   `forward(query: str) -> str`: Takes a search query as input and performs a web search using the DuckDuckGo API. Returns the search results as a string.

**Notable Dependencies:**

The class depends on the `duckduckgo_search` package, which needs to be installed via pip: `pip install duckduckgo-search`. The tool also attempts to import the `DDGS` (DuckDuckGo Search) module from this package.


key: chroma:document
string_value:
CODE:

class VisitWebpageTool(Tool):
    name = "visit_webpage"
    description = "Visits a webpage at the given url and returns its content as a markdown string."
    inputs = {
        "url": {
            "type": "string",
            "description": "The url of the webpage to visit.",
        }
    }
    output_type = "string"

    def forward(self, url: str) -> str:
        try:
            from markdownify import markdownify
        except ImportError:
            raise ImportError(
                "You must install package `markdownify` to run this tool: for instance run `pip install markdownify`."
            )
        try:
            # Send a GET request to the URL
            response = requests.get(url)
            response.raise_for_status()  # Raise an exception for bad status codes

            # Convert the HTML content to Markdown
            markdown_content = markdownify(response.text).strip()

            # Remove multiple line breaks
            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)

            return markdown_content

        except RequestException as e:
            return f"Error fetching the webpage: {str(e)}"
        except Exception as e:


SUMMARY:
**Summary**

The `VisitWebpageTool` class is a custom tool designed to fetch and convert HTML content of a given webpage into Markdown format. Its primary purpose is to visit a webpage at the specified URL, extract its content, and return it as a Markdown string.

**Key Implementation Details**

*   Fetches HTML content using `requests.get()`.
*   Converts HTML content to Markdown using the `markdownify` package.
*   Removes multiple line breaks from the Markdown content.

**Notable Dependencies/Requirements**

*   The `markdownify` package, which can be installed via pip (`pip install markdownify`) or specified in the tool's initialization.

**Method Signatures and Interfaces**

*   `forward(url: str) -> str`: Takes a string URL as input and returns a Markdown string. It handles exceptions for request errors and other unexpected issues.

Here is a simple representation of a RAG system to display the class code summary:

```
 VisitWebpageTool
  -------------

**Overview**
-----------
*   Custom tool to fetch and convert HTML content to Markdown format

**Key Implementation Details**
-------------------------------

### Key Steps

1.  Fetch HTML content using `requests.get()`.
2.  Convert HTML content to Markdown using `markdownify()`.
3.  Remove multiple line breaks from the Markdown content.

### Errors

*   Request errors (handled with `try/except` block)
*   Unexpected errors (reported as an error message)

**Dependencies**
-----------------

*   The `markdownify` package, which can be installed via pip (`pip install markdownify`) or specified in the tool's initialization.
```


key: chroma:document
string_value:
CODE:

class DocumentQuestionAnsweringTool(PipelineTool):
    default_checkpoint = "naver-clova-ix/donut-base-finetuned-docvqa"
    description = "This is a tool that answers a question about an document (pdf). It returns a string that contains the answer to the question."
    name = "document_qa"
    pre_processor_class = AutoProcessor
    model_class = VisionEncoderDecoderModel

    inputs = {
        "document": {
            "type": "image",
            "description": "The image containing the information. Can be a PIL Image or a string path to the image.",
        },
        "question": {"type": "string", "description": "The question in English"},
    }
    output_type = "string"

    def __init__(self, *args, **kwargs):
        if not is_vision_available():
            raise ValueError("Pillow must be installed to use the DocumentQuestionAnsweringTool.")

        super().__init__(*args, **kwargs)

    def encode(self, document: "Image", question: str):
        task_prompt = "<s_docvqa><s_question>{user_input}</s_question><s_answer>"
        prompt = task_prompt.replace("{user_input}", question)
        decoder_input_ids = self.pre_processor.tokenizer(
            prompt, add_special_tokens=False, return_tensors="pt"
        ).input_ids
        if isinstance(document, str):
            img = Image.open(document).convert("RGB")
            img_array = np.array(img).transpose(2, 0, 1)
            document = torch.from_numpy(img_array)
        pixel_values = self.pre_processor(document, return_tensors="pt").pixel_values

        return {"decoder_input_ids": decoder_input_ids, "pixel_values": pixel_values}

    def forward(self, inputs):
        return self.model.generate(
            inputs["pixel_values"].to(self.device),
            decoder_input_ids=inputs["decoder_input_ids"].to(self.device),
            max_length=self.model.decoder.config.max_position_embeddings,
            early_stopping=True,
            pad_token_id=self.pre_processor.tokenizer.pad_token_id,
            eos_token_id=self.pre_processor.tokenizer.eos_token_id,
            use_cache=True,
            num_beams=1,
            bad_words_ids=[[self.pre_processor.tokenizer.unk_token_id]],
            return_dict_in_generate=True,
        ).sequences

    def decode(self, outputs):
        sequence = self.pre_processor.batch_decode(outputs)[0]
        sequence = sequence.replace(self.pre_processor.tokenizer.eos_token, "")
        sequence = sequence.replace(self.pre_processor.tokenizer.pad_token, "")
        sequence = re.sub(r"<.*?>", "", sequence, count=1).strip()  # remove first task start token
        sequence = self.pre_processor.token2json(sequence)



SUMMARY:
**Concise Summary**

The `DocumentQuestionAnsweringTool` class is a pipeline tool for document question answering. Its primary purpose is to take an image document and a question as input, encode them into model-friendly formats, and use a vision encoder-decoder model to generate a response.

**Key Implementation Details**

* Models:
	+ `VisionEncoderDecoderModel`: uses pre-trained ViT model
	+ `AutoProcessor`: preprocesses inputs for the model
* Input/Output Types:
	+ Inputs: `document` (image) and `question` (string)
	+ Outputs: response as a string ( `output_type = "string"` )
* Key Method Signatures:
	+ `encode(document, question)` : encodes input data into model-friendly formats
	  - `prompt`: task prompt with user input
	- `pixel_values` and `decoder_input_ids`
	+ `forward(inputs)` : passes encoded inputs through the vision encoder-decoder model
	  - uses `max_length`, `early_stopping`, etc. for model generation
* Dependencies:
	+ `PIL Image` for image processing
	+ `torch` for tensor operations

**Notable Requirements**

* `Pillow` or `Image` must be installed for image support
* Model checkpoint: `naver-clova-ix/donut-base-finetuned-docvqa`


key: chroma:document
string_value:
CODE:

class ImageQuestionAnsweringTool(PipelineTool):
    default_checkpoint = "dandelin/vilt-b32-finetuned-vqa"
    description = (
        "This is a tool that answers a question about an image. It "
        "returns a text that is the answer to the question."
    )
    name = "image_qa"
    pre_processor_class = AutoProcessor
    model_class = AutoModelForVisualQuestionAnswering

    inputs = {
        "image": {
            "type": "image",
            "description": "The image containing the information. Can be a PIL Image or a string path to the image.",
        },
        "question": {"type": "string", "description": "The question in English"},
    }
    output_type = "string"

    def __init__(self, *args, **kwargs):
        requires_backends(self, ["vision"])
        super().__init__(*args, **kwargs)

    def encode(self, image: "Image", question: str):
        return self.pre_processor(image, question, return_tensors="pt")

    def forward(self, inputs):
        with torch.no_grad():
            return self.model(**inputs).logits

    def decode(self, outputs):
        idx = outputs.argmax(-1).item()


SUMMARY:
**Class Summary: Image Question Answering Tool**

The `ImageQuestionAnsweringTool` class is a pipeline tool that answers visual questions. Its primary function is to:

* Take in an image and a question as inputs
* Use pre-processing and model inference to generate the answer
* Return the answer as a string

**Key Implementation Details:**

* Uses Hugging Face's `AutoModelForVisualQuestionAnswering` for model inference
* Inferred logits from the model are converted back to labels using `id2label` mapping
* Pre-processing is handled by `AutoProcessor`

**Notable Requirements and Dependencies:**

* Requires the `vision` backend
* Uses PyTorch for tensor operations
* Depends on Hugging Face's Transformers library for model inference

In summary, this class provides a simple tool for answering visual questions using a pre-trained model, with focus on robust image processing and answer decoding.


key: chroma:document
string_value:
CODE:

class TextToSpeechTool(PipelineTool):
    default_checkpoint = "microsoft/speecht5_tts"
    description = (
        "This is a tool that reads an English text out loud. It returns a waveform object containing the sound."
    )
    name = "text_to_speech"
    pre_processor_class = SpeechT5Processor
    model_class = SpeechT5ForTextToSpeech
    post_processor_class = SpeechT5HifiGan

    inputs = {"text": {"type": "string", "description": "The text to read out loud (in English)"}}
    output_type = "audio"

    def setup(self):
        if self.post_processor is None:
            self.post_processor = "microsoft/speecht5_hifigan"
        super().setup()

    def encode(self, text, speaker_embeddings=None):
        inputs = self.pre_processor(text=text, return_tensors="pt", truncation=True)

        if speaker_embeddings is None:
            if not is_datasets_available():
                raise ImportError("Datasets needs to be installed if not passing speaker embeddings.")

            embeddings_dataset = load_dataset(
                "Matthijs/cmu-arctic-xvectors", split="validation", trust_remote_code=True
            )
            speaker_embeddings = torch.tensor(embeddings_dataset[7305]["xvector"]).unsqueeze(0)

        return {"input_ids": inputs["input_ids"], "speaker_embeddings": speaker_embeddings}

    def forward(self, inputs):
        with torch.no_grad():
            return self.model.generate_speech(**inputs)

    def decode(self, outputs):
        with torch.no_grad():


SUMMARY:
**Class Summary: TextToSpeechTool**

The `TextToSpeechTool` class is a pipeline-based tool for converting English text to spoken audio. Its primary purpose is to use the Microsoft Speech T5 model to generate audio waveforms.

### Key Functionality:

*   **Input and Output Processing**: The class processes input text through the `SpeechT5Processor`, generates audio waveforms using the `SpeechT5ForTextToSpeech` model, and decodes them using the `SpeechT5HifiGan` post-processor.
*   **Speaker Embeddings Integration**: Optional speaker embeddings can be passed to the tool, retrieving them from the 'Matthijs/cmu-arctic-xvectors' dataset if not already provided.

### Implementation Details:

*   The `setup` method initializes the `post_processor_class` with a default value of `"microsoft/speecht5_hifigan"` unless it's set manually.
*   The `encode` method preprocesses input text using the `pre_processor`, optionally loads speaker embeddings from a dataset if not provided.

### Notable Dependencies and Requirements:

*   Microsoft Speech T5 model
*   Torch library
*   Datasets library (for retrieving 'Matthijs/cmu-arctic-xvectors' dataset)
*   Speakers Embeddings ('Matthijs/cmu-arctic-xvectors') 

    This code uses the `is_datasets_available()` function to check if the datasets library is installed and available for use with external dependencies.


key: chroma:document
string_value:
CODE:

def is_empty_line(line: str) -> bool:
    """
    Determines whether a line is empty or not.
    """


SUMMARY:
**Summary: `is_empty_line` Function**

This function determines whether a given string is an empty line by checking its length and whitespace characters using the `isspace()` method.

* **Purpose:** Determine if a line is empty
* **Functionality:** Returns `True` for empty lines (length 0 or with only whitespace)
* **Implementation Details:**
	+ Uses `len(line) == 0` to check for literal zero-length strings
	+ Employs `line.isspace()` to consider whitespace characters as part of an "empty" line


key: chroma:document
string_value:
CODE:

def find_indent(line: str) -> int:
    """
    Returns the number of spaces that start a line indent.
    """
    search = re.search(r"^(\s*)(?:\S|$)", line)
    if search is None:
        return 0


SUMMARY:
**Summary:**

The `find_indent` function calculates the number of leading spaces that start a line indent in a given input string (`line`). It uses a regular expression to search for a match at the beginning of the line, capturing only the whitespace characters until the first non-whitespace character is encountered. The function returns the length of this captured whitespace sub-string.

**Key Details:**

- **Regular Expression Pattern:** `^(\s*)(?:\S|$)`
- **Return Type:** Integer (`int`)

- **Notable Functions/Classes:**
  - `re.search()`: used to search for a pattern in a string
- **Dependencies/Requirements:**
  - `re` (regular expression) module


key: chroma:document
string_value:
CODE:

def parse_module_content(content: str) -> List[str]:
    """
    Parse the content of a module in the list of objects it defines.

    Args:
        content (`str`): The content to parse

    Returns:
        `List[str]`: The list of objects defined in the module.
    """
    objects = []
    current_object = []
    lines = content.split("\n")
    # Doc-styler takes everything between two triple quotes in docstrings, so we need a fake """ here to go with this.
    end_markers = [")", "]", "}", '"""']

    for line in lines:
        # End of an object
        is_valid_object = len(current_object) > 0
        if is_valid_object and len(current_object) == 1:
            is_valid_object = not current_object[0].startswith("# Copied from")
        if not is_empty_line(line) and find_indent(line) == 0 and is_valid_object:
            # Closing parts should be included in current object
            if line in end_markers:
                current_object.append(line)
                objects.append("\n".join(current_object))
                current_object = []
            else:
                objects.append("\n".join(current_object))
                current_object = [line]
        else:
            current_object.append(line)

    # Add last object
    if len(current_object) > 0:
        objects.append("\n".join(current_object))



SUMMARY:
**Summary**

### `parse_module_content` Function

#### Purpose and Functionality

Parses the content of a module in a list of defined objects.

*   Takes a string `content` as input and returns a list of strings, where each string represents an object in the module.
*   Uses newline characters to separate objects and handles docstrings with triple quotes.

#### Key Implementation Details

*   Iterates over each line in `content`, checking for:
    *   Object boundaries: lines that start with indentation or begin a new object.
    *   Docstring handling: special cases for closing marks (`"}`, `"}``) and ignoring lines starting with `" # Copied from".
*   Appends lines to the current object, joining them with newline characters when transitioning between objects.

#### Notable Dependencies

*   `find_indent(line)` function to determine indentation for each line.
*   `is_empty_line(line)` function to check if a line is empty (not provided in this code snippet).
*   External dependencies not specified in the provided code.

### Code Organization and Notes

The code uses a simple loop to iterate over each line in the input string, applying specific logic to determine object boundaries and handle docstrings. The use of newline characters to separate objects ensures that the output is readable as a list of strings.


key: chroma:document
string_value:
CODE:

def extract_block(content: str, indent_level: int = 0) -> str:
    """Return the first block in `content` with the indent level `indent_level`.

    The first line in `content` should be indented at `indent_level` level, otherwise an error will be thrown.

    This method will immediately stop the search when a (non-empty) line with indent level less than `indent_level` is
    encountered.

    Args:
        content (`str`): The content to parse
        indent_level (`int`, *optional*, default to 0): The indent level of the blocks to search for

    Returns:
        `str`: The first block in `content` with the indent level `indent_level`.
    """
    current_object = []
    lines = content.split("\n")
    # Doc-styler takes everything between two triple quotes in docstrings, so we need a fake """ here to go with this.
    end_markers = [")", "]", "}", '"""']

    for idx, line in enumerate(lines):
        if idx == 0 and indent_level > 0 and not is_empty_line(line) and find_indent(line) != indent_level:
            raise ValueError(
                f"When `indent_level > 0`, the first line in `content` should have indent level {indent_level}. Got "
                f"{find_indent(line)} instead."
            )

        if find_indent(line) < indent_level and not is_empty_line(line):
            break

        # End of an object
        is_valid_object = len(current_object) > 0
        if (
            not is_empty_line(line)
            and not line.endswith(":")
            and find_indent(line) == indent_level
            and is_valid_object
        ):
            # Closing parts should be included in current object
            if line.lstrip() in end_markers:
                current_object.append(line)
            return "\n".join(current_object)
        else:
            current_object.append(line)

    # Add last object
    if len(current_object) > 0:


SUMMARY:
**Summary: `extract_block` Function**

**Purpose:** Returns the first block in `content` with indent level matching `indent_level`.

**Functionality:**

* Splits input `content` into lines and iterates through them
* Uses `find_indent` to determine line indentation levels
* Continuously searches for a block with matching indent level, stopping when an incompatible line is found

**Key Implementation Details:**

* Uses list `current_object` to accumulate parts of the block until a complete one is identified
* Checks for `is_empty_line`, `end_markers`, and `find_indent` conditions to decide whether to add lines to `current_object` or return the block

**Parameters and Returns:**

* `$content`: input string to parse ($str)
* `$indent_level`: indent level of target blocks (default 0, $int)

Returns the first matched block as a string (`$str`). Throws an error if initial line does not match specified `indent_level`.


key: chroma:document
string_value:
CODE:

def add_content_to_text(
    text: str,
    content: str,
    add_after: Optional[Union[str, Pattern]] = None,
    add_before: Optional[Union[str, Pattern]] = None,
    exact_match: bool = False,
) -> str:
    """
    A utility to add some content inside a given text.

    Args:
       text (`str`): The text in which we want to insert some content.
       content (`str`): The content to add.
       add_after (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added after the first instance matching it.
       add_before (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added before the first instance matching it.
       exact_match (`bool`, *optional*, defaults to `False`):
           A line is considered a match with `add_after` or `add_before` if it matches exactly when `exact_match=True`,
           otherwise, if `add_after`/`add_before` is present in the line.

    <Tip warning={true}>

    The arguments `add_after` and `add_before` are mutually exclusive, and one exactly needs to be provided.

    </Tip>

    Returns:
        `str`: The text with the new content added if a match was found.
    """
    if add_after is None and add_before is None:
        raise ValueError("You need to pass either `add_after` or `add_before`")
    if add_after is not None and add_before is not None:
        raise ValueError("You can't pass both `add_after` or `add_before`")
    pattern = add_after if add_before is None else add_before

    def this_is_the_line(line):
        if isinstance(pattern, Pattern):
            return pattern.search(line) is not None
        elif exact_match:
            return pattern == line
        else:
            return pattern in line

    new_lines = []
    for line in text.split("\n"):
        if this_is_the_line(line):
            if add_before is not None:
                new_lines.append(content)
            new_lines.append(line)
            if add_after is not None:
                new_lines.append(content)
        else:
            new_lines.append(line)



SUMMARY:
**Summary:**

The `add_content_to_text` function adds content to a given text, either before or after specific lines based on a given pattern.

**Main Purpose and Functionality:**

- Inserts content into a text
- Can insert content at specific lines based on a given pattern (either `add_before` or `add_after`)
- Handles exact matches and partial matches separately

**Key Implementation Details:**

* Uses regular expressions (`Pattern`) for pattern matching
* Mutually exclusive arguments `add_after` and `add_before`
* `exact_match` parameter influences the behavior of pattern matching

**Notable Dependencies or Requirements:**

* Import of `Optional`, `Union`, `Pattern`, and related classes/function from `...`

Note that some dependencies were not specified, so I assumed a mention of general library imports. Adjust according to actual library usage if necessary.


key: chroma:document
string_value:
CODE:

def add_content_to_file(
    file_name: Union[str, os.PathLike],
    content: str,
    add_after: Optional[Union[str, Pattern]] = None,
    add_before: Optional[Union[str, Pattern]] = None,
    exact_match: bool = False,
):
    """
    A utility to add some content inside a given file.

    Args:
       file_name (`str` or `os.PathLike`): The name of the file in which we want to insert some content.
       content (`str`): The content to add.
       add_after (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added after the first instance matching it.
       add_before (`str` or `Pattern`):
           The pattern to test on a line of `text`, the new content is added before the first instance matching it.
       exact_match (`bool`, *optional*, defaults to `False`):
           A line is considered a match with `add_after` or `add_before` if it matches exactly when `exact_match=True`,
           otherwise, if `add_after`/`add_before` is present in the line.

    <Tip warning={true}>

    The arguments `add_after` and `add_before` are mutually exclusive, and one exactly needs to be provided.

    </Tip>
    """
    with open(file_name, "r", encoding="utf-8") as f:
        old_content = f.read()

    new_content = add_content_to_text(
        old_content, content, add_after=add_after, add_before=add_before, exact_match=exact_match
    )

    with open(file_name, "w", encoding="utf-8") as f:


SUMMARY:
Here's a concise and technical summary of the `add_content_to_file` function:

**Function:** `add_content_to_file`

**Purpose:** Inserts specified content before or after matched lines in a given file.

**Parameters:**

*   `file_name`: Path to the file where content is inserted (`str` or `os.PathLike`).
*   `content`: The content to be added to the file (`str`).
*   `add_after`: Pattern for inserting content after matching line (optional, `str` or `Pattern`, exclusive with `add_before`).
*   `add_before`: Pattern for inserting content before matching line (optional, `str` or `Pattern`, exclusive with `add_after`).
*   `exact_match`: Flag indicating whether a match requires an exact alignment of the pattern to the line (`bool`, default: `False`).

**Behavior:**

1.  Reads existing file content.
2.  Applies specified insertion logic using `add_content_to_text`.
3.  Appends modified content back to the specified file.

 **Key Implementation Details:** 

*   Utilizes regular expressions for pattern matching (`Pattern`).
*   Enforces mutual exclusivity between `add_after` and `add_before`.


key: chroma:document
string_value:
CODE:

def replace_model_patterns(
    text: str, old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns
) -> Tuple[str, str]:
    """
    Replace all patterns present in a given text.

    Args:
        text (`str`): The text to treat.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.

    Returns:
        `Tuple(str, str)`: A tuple of with the treated text and the replacement actually done in it.
    """
    # The order is crucially important as we will check and replace in that order. For instance the config probably
    # contains the camel-cased named, but will be treated before.
    attributes_to_check = ["config_class"]
    # Add relevant preprocessing classes
    for attr in ["tokenizer_class", "image_processor_class", "feature_extractor_class", "processor_class"]:
        if getattr(old_model_patterns, attr) is not None and getattr(new_model_patterns, attr) is not None:
            attributes_to_check.append(attr)

    # Special cases for checkpoint and model_type
    if old_model_patterns.checkpoint not in [old_model_patterns.model_type, old_model_patterns.model_lower_cased]:
        attributes_to_check.append("checkpoint")
    if old_model_patterns.model_type != old_model_patterns.model_lower_cased:
        attributes_to_check.append("model_type")
    else:
        text = re.sub(
            rf'(\s*)model_type = "{old_model_patterns.model_type}"',
            r'\1model_type = "[MODEL_TYPE]"',
            text,
        )

    # Special case when the model camel cased and upper cased names are the same for the old model (like for GPT2) but
    # not the new one. We can't just do a replace in all the text and will need a special regex
    if old_model_patterns.model_upper_cased == old_model_patterns.model_camel_cased:
        old_model_value = old_model_patterns.model_upper_cased
        if re.search(rf"{old_model_value}_[A-Z_]*[^A-Z_]", text) is not None:
            text = re.sub(rf"{old_model_value}([A-Z_]*)([^a-zA-Z_])", r"[MODEL_UPPER_CASED]\1\2", text)
    else:
        attributes_to_check.append("model_upper_cased")

    attributes_to_check.extend(["model_camel_cased", "model_lower_cased", "model_name"])

    # Now let's replace every other attribute by their placeholder
    for attr in attributes_to_check:
        text = text.replace(getattr(old_model_patterns, attr), ATTRIBUTE_TO_PLACEHOLDER[attr])

    # Finally we can replace the placeholder byt the new values.
    replacements = []
    for attr, placeholder in ATTRIBUTE_TO_PLACEHOLDER.items():
        if placeholder in text:
            replacements.append((getattr(old_model_patterns, attr), getattr(new_model_patterns, attr)))
            text = text.replace(placeholder, getattr(new_model_patterns, attr))

    # If we have two inconsistent replacements, we don't return anything (ex: GPT2->GPT_NEW and GPT2->GPTNew)
    old_replacement_values = [old for old, new in replacements]
    if len(set(old_replacement_values)) != len(old_replacement_values):
        return text, ""

    replacements = simplify_replacements(replacements)
    replacements = [f"{old}->{new}" for old, new in replacements]


SUMMARY:
**Function Summary: `replace_model_patterns`**

Replaces patterns in a given text with placeholders and subsequently replaces the placeholders with actual values.

### Main Purpose and Functionality

The function takes three inputs:
- `text`: The text to replace patterns in
- `old_model_patterns`: Patterns for the old model
- `new_model_patterns`: Patterns for the new model
It returns a tuple containing the treated text and a string of replacement made in it.

### Key Implementation Details

*   Replaces all attributes specified in `attributes_to_check` with placeholders.
*   Uses special cases to handle attribute names like `model_type`, `checkpoint` and camel-cased/camell cased upper named attribute.
*   Calculates replacements between the old model and new model patterns.
*   Applies replacements using a preprocessing rule.

### Parameters

-  `text`: The text to replace patterns in
- Old and New pattern for models 

### Return Value

Returns the treated text with replacement, made string


key: chroma:document
string_value:
CODE:

def simplify_replacements(replacements):
    """
    Simplify a list of replacement patterns to make sure there are no needless ones.

    For instance in the sequence "Bert->BertNew, BertConfig->BertNewConfig, bert->bert_new", the replacement
    "BertConfig->BertNewConfig" is implied by "Bert->BertNew" so not needed.

    Args:
        replacements (`List[Tuple[str, str]]`): List of patterns (old, new)

    Returns:
        `List[Tuple[str, str]]`: The list of patterns simplified.
    """
    if len(replacements) <= 1:
        # Nothing to simplify
        return replacements

    # Next let's sort replacements by length as a replacement can only "imply" another replacement if it's shorter.
    replacements.sort(key=lambda x: len(x[0]))

    idx = 0
    while idx < len(replacements):
        old, new = replacements[idx]
        # Loop through all replacements after
        j = idx + 1
        while j < len(replacements):
            old_2, new_2 = replacements[j]
            # If the replacement is implied by the current one, we can drop it.
            if old_2.replace(old, new) == new_2:
                replacements.pop(j)
            else:
                j += 1
        idx += 1



SUMMARY:
**Function Summary**

The `simplify_replacements` function simplifies a list of replacement patterns by removing implied replacements. It takes a list of tuples (old, new) and returns the simplified list.

**Key Details**

* The function first sorts the replacements based on length, allowing shorter replacements to imply longer ones.
* It iterates through each replacement, checking if it implies any subsequent replacements by searching for occurrences of the old string in the new string. If an implied replacement is found, it is removed from the list.

**Implementation Context**

This function appears to be part of a text normalization or tokenization process, potentially used in natural language processing (NLP) applications. The function assumes the input list contains valid replacement patterns and may not handle invalid inputs.

**Notable Dependencies**

* `List` and `Tuple` from Python's built-in `collections` module
* No external dependencies mentioned

**Interface Inheritance/Implementation**

* This function implements a custom sorting behavior by using a `lambda` function as the sorting key.
* It relies on the `replace` method of Python strings to check for implied replacements.


key: chroma:document
string_value:
CODE:

def get_module_from_file(module_file: Union[str, os.PathLike]) -> str:
    """
    Returns the module name corresponding to a module file.
    """
    full_module_path = Path(module_file).absolute()
    module_parts = full_module_path.with_suffix("").parts

    # Find the first part named transformers, starting from the end.
    idx = len(module_parts) - 1
    while idx >= 0 and module_parts[idx] != "transformers":
        idx -= 1
    if idx < 0:
        raise ValueError(f"{module_file} is not a transformers module.")



SUMMARY:
**Summary:**

`get_module_from_file` function returns the module name corresponding to a module file.

**Key Features:**

* Takes an absolute path of a module file as input.
* Extracts module parts by removing the suffix and splitting into components.
* Finds the last "transformers" component and joins all preceding parts to form the module name.
* Retsunns the formatted module name as a string.
 
**Notable Details:**

* Throws `ValueError` if no "transformers" component is found in the path.

No external dependencies are reported.


key: chroma:document
string_value:
CODE:

def remove_attributes(obj, target_attr):
    """Remove `target_attr` in `obj`."""
    lines = obj.split(os.linesep)

    target_idx = None
    for idx, line in enumerate(lines):
        # search for assignment
        if line.lstrip().startswith(f"{target_attr} = "):
            target_idx = idx
            break
        # search for function/method definition
        elif line.lstrip().startswith(f"def {target_attr}("):
            target_idx = idx
            break

    # target not found
    if target_idx is None:
        return obj

    line = lines[target_idx]
    indent_level = find_indent(line)
    # forward pass to find the ending of the block (including empty lines)
    parsed = extract_block("\n".join(lines[target_idx:]), indent_level)
    num_lines = len(parsed.split("\n"))
    for idx in range(num_lines):
        lines[target_idx + idx] = None

    # backward pass to find comments or decorator
    for idx in range(target_idx - 1, -1, -1):
        line = lines[idx]
        if (line.lstrip().startswith("#") or line.lstrip().startswith("@")) and find_indent(line) == indent_level:
            lines[idx] = None
        else:
            break

    new_obj = os.linesep.join([x for x in lines if x is not None])



SUMMARY:
**Concise Summary**

`remove_attributes` Function:

Removes the specified attribute (`target_attr`) from an object (`obj`). 
The function performs a forward pass to find the start of the attribute's block and a backward pass to detect any comments or decorators before it.

**Key Functions and Details:**

* Splits input object into lines
* Searches for assignment or method definition containing `target_attr`
* Uses indentation level from target line to extract surrounding code
* Performs forward pass to find attribute block end, followed by backward pass to remove comments or decorators
* Returns modified object without the specified attribute

**Notable Dependencies:**

* `os` for line separator handling
* `find_indent` and `extract_block` functions (not shown in this snippet)
 
**Implementation Notes:**
This function appears to be implementing a custom decorator removal logic, likely part of a larger framework or library. The implementation is complex due to its attempt to handle both explicit assignment and method definition cases for attribute removal.


key: chroma:document
string_value:
CODE:

def duplicate_module(
    module_file: Union[str, os.PathLike],
    old_model_patterns: ModelPatterns,
    new_model_patterns: ModelPatterns,
    dest_file: Optional[str] = None,
    add_copied_from: bool = True,
    attrs_to_remove: List[str] = None,
):
    """
    Create a new module from an existing one and adapting all function and classes names from old patterns to new ones.

    Args:
        module_file (`str` or `os.PathLike`): Path to the module to duplicate.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        dest_file (`str` or `os.PathLike`, *optional*): Path to the new module.
        add_copied_from (`bool`, *optional*, defaults to `True`):
            Whether or not to add `# Copied from` statements in the duplicated module.
    """
    if dest_file is None:
        dest_file = str(module_file).replace(
            old_model_patterns.model_lower_cased, new_model_patterns.model_lower_cased
        )

    with open(module_file, "r", encoding="utf-8") as f:
        content = f.read()

    content = re.sub(r"# Copyright (\d+)\s", f"# Copyright {CURRENT_YEAR} ", content)
    objects = parse_module_content(content)

    # Loop and treat all objects
    new_objects = []
    for obj in objects:
        special_pattern = False
        for pattern, attr in SPECIAL_PATTERNS.items():
            if pattern in obj:
                obj = obj.replace(getattr(old_model_patterns, attr), getattr(new_model_patterns, attr))
                new_objects.append(obj)
                special_pattern = True
                break

        if special_pattern:
            continue

        # Regular classes functions
        old_obj = obj
        obj, replacement = replace_model_patterns(obj, old_model_patterns, new_model_patterns)
        has_copied_from = re.search(r"^#\s+Copied from", obj, flags=re.MULTILINE) is not None
        if add_copied_from and not has_copied_from and _re_class_func.search(obj) is not None and len(replacement) > 0:
            # Copied from statement must be added just before the class/function definition, which may not be the
            # first line because of decorators.
            module_name = get_module_from_file(module_file)
            old_object_name = _re_class_func.search(old_obj).groups()[0]
            obj = add_content_to_text(
                obj, f"# Copied from {module_name}.{old_object_name} with {replacement}", add_before=_re_class_func
            )
        # In all cases, we remove Copied from statement with indent on methods.
        obj = re.sub("\n[ ]+# Copied from [^\n]*\n", "\n", obj)

        new_objects.append(obj)

    content = "\n".join(new_objects)
    # Remove some attributes that we don't want to copy to the new file(s)
    if attrs_to_remove is not None:
        for attr in attrs_to_remove:
            content = remove_attributes(content, target_attr=attr)

    with open(dest_file, "w", encoding="utf-8") as f:


SUMMARY:
**Function Summary: `duplicate_module`**

### Purpose and Functionality

Duplicates an existing module by renaming all classes and functions from the old model patterns to new ones, while preserving copyright information. The function takes an optional destination file path, adds "Copied from" statements if specified, and allows optional removal of certain attributes.

### Key Implementation Details

- Uses regular expressions to replace model patterns in the module content.
- Compares pattern names using `SPECIAL_PATTERNS` dictionary.
- Replaces copyright information in line comments.
- Includes or excludes "Copied from" statements based on user input and pattern visibility.

### Notable Dependencies

- `os`: for file path manipulation
- `re` (regular expressions): for string pattern matching and replacement
- `ModelPatterns`: class containing old and new model patterns
- `_re_class_func` and `_add_content_to_text`: custom functions for regular expression matching and string modification, respectively.

### Interface Inheritance

None.


key: chroma:document
string_value:
CODE:

def filter_framework_files(
    files: List[Union[str, os.PathLike]], frameworks: Optional[List[str]] = None
) -> List[Union[str, os.PathLike]]:
    """
    Filter a list of files to only keep the ones corresponding to a list of frameworks.

    Args:
        files (`List[Union[str, os.PathLike]]`): The list of files to filter.
        frameworks (`List[str]`, *optional*): The list of allowed frameworks.

    Returns:
        `List[Union[str, os.PathLike]]`: The list of filtered files.
    """
    if frameworks is None:
        frameworks = get_default_frameworks()

    framework_to_file = {}
    others = []
    for f in files:
        parts = Path(f).name.split("_")
        if "modeling" not in parts:
            others.append(f)
            continue
        if "tf" in parts:
            framework_to_file["tf"] = f
        elif "flax" in parts:
            framework_to_file["flax"] = f
        else:
            framework_to_file["pt"] = f



SUMMARY:
**Function Summary: Filter Framework Files**

### Purpose and Functionality

The `filter_framework_files` function filters a list of files to only keep the ones corresponding to a specified list of frameworks. It returns a new list containing the matching files.

### Key Implementation Details

* Maps framework names to file paths using a dictionary (`framework_to_file`)
* Uses `Path` and `split` methods to extract framework indicators from file names
* Separates files into two lists: one for matching frameworks and another for other frameworks/others (e.g., "modeling" but not a recognized framework)

### Dependencies

* `List`: used as type hint for input parameters and return value
* `Optional`: used as type hint for optionalFrameworks parameter
* `get_default_frameworks` function: returns the list of default frameworks if none are provided
* `Path`: used to split file names into parts

### Returns

A list of filtered files, containing files corresponding to the specified frameworks and others (if present).


key: chroma:document
string_value:
CODE:

def get_model_files(model_type: str, frameworks: Optional[List[str]] = None) -> Dict[str, Union[Path, List[Path]]]:
    """
    Retrieves all the files associated to a model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the model files corresponding to the passed frameworks.

    Returns:
        `Dict[str, Union[Path, List[Path]]]`: A dictionary with the following keys:
        - **doc_file** -- The documentation file for the model.
        - **model_files** -- All the files in the model module.
        - **test_files** -- The test files for the model.
    """
    module_name = model_type_to_module_name(model_type)

    model_module = TRANSFORMERS_PATH / "models" / module_name
    model_files = list(model_module.glob("*.py"))
    model_files = filter_framework_files(model_files, frameworks=frameworks)

    doc_file = REPO_PATH / "docs" / "source" / "en" / "model_doc" / f"{model_type}.md"

    # Basic pattern for test files
    test_files = [
        f"test_modeling_{module_name}.py",
        f"test_modeling_tf_{module_name}.py",
        f"test_modeling_flax_{module_name}.py",
        f"test_tokenization_{module_name}.py",
        f"test_image_processing_{module_name}.py",
        f"test_feature_extraction_{module_name}.py",
        f"test_processor_{module_name}.py",
    ]
    test_files = filter_framework_files(test_files, frameworks=frameworks)
    # Add the test directory
    test_files = [REPO_PATH / "tests" / "models" / module_name / f for f in test_files]
    # Filter by existing files
    test_files = [f for f in test_files if f.exists()]



SUMMARY:
**Summary**

The `get_model_files` function retrieves files associated with a given model type. It:

1. **Filters data**: Returns a dictionary containing the documentation file and model files (`*.py`) specified by the model type, or filtered to include only files corresponding to specific frameworks (if provided).
2. **Design choice**: Uses the `model_type_to_module_name` function to map the model type to its corresponding module name.
3. **Key implementation details**:
   - Uses regular expressions to filter framework-specific files in `model_files`.
   - Appends the test file patterns with the module name and filters by existing files using `exists()`.
4. **Notable dependencies**: References:
  * `REPO_PATH` and `TRANSFORMERSPATH`
  * `filter_framework_files()` function
5. **External dependencies**:
  * `pathlib` (used implicitly through `Path` classes)
* 
 **Assumed variables and interfaces**:

 - `model_type_to_module_name`: Function mapping model types to their corresponding module names.
 -(Function signature) is assumed from this snippet in line `"model_type (`str`): ...`

This code is part of a project managing various models, likely for natural language processing. The function provides flexibility by allowing filtering of files based on specific frameworks.


key: chroma:document
string_value:
CODE:

def find_base_model_checkpoint(
    model_type: str, model_files: Optional[Dict[str, Union[Path, List[Path]]]] = None
) -> str:
    """
    Finds the model checkpoint used in the docstrings for a given model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        model_files (`Dict[str, Union[Path, List[Path]]`, *optional*):
            The files associated to `model_type`. Can be passed to speed up the function, otherwise will be computed.

    Returns:
        `str`: The checkpoint used.
    """
    if model_files is None:
        model_files = get_model_files(model_type)
    module_files = model_files["model_files"]
    for fname in module_files:
        if "modeling" not in str(fname):
            continue

        with open(fname, "r", encoding="utf-8") as f:
            content = f.read()
            if _re_checkpoint_for_doc.search(content) is not None:
                checkpoint = _re_checkpoint_for_doc.search(content).groups()[0]
                # Remove quotes
                checkpoint = checkpoint.replace('"', "")
                checkpoint = checkpoint.replace("'", "")
                return checkpoint

    # TODO: Find some kind of fallback if there is no _CHECKPOINT_FOR_DOC in any of the modeling file.


SUMMARY:
**Function Summary**

`find_base_model_checkpoint`: Retrieves the base model checkpoint used for a given `model_type`. If `model_files` are not provided, it computes and caches them.

**Key Details:**

* **Argument:** `model_type` (str) - Must match a valid model type.
* **Implementation:**
	+ Reads file contents for each "modeling" file associated with the `model_type`.
	+ Uses regular expression `_re_checkpoint_for_doc` to search for the checkpoint pattern in content.
	+ Returns the found checkpoint string after removing quotes.
* **Return Value:** The base model checkpoint used, or an empty string if not found.

**Notable Dependencies:**

* `_re_checkpoint_for_doc` - A regular expression pattern for matching checkpoints (function/method not shown).
* `get_model_files(model_type)`, `model_files`, and `module_files` - Functions/methods not shown that interact with the codebase.


key: chroma:document
string_value:
CODE:

def get_default_frameworks():
    """
    Returns the list of frameworks (PyTorch, TensorFlow, Flax) that are installed in the environment.
    """
    frameworks = []
    if is_torch_available():
        frameworks.append("pt")
    if is_tf_available():
        frameworks.append("tf")
    if is_flax_available():
        frameworks.append("flax")


SUMMARY:
**Summary:**

The `get_default_frameworks()` function returns a list of installed frameworks (PyTorch, TensorFlow, Flax) in the environment. It checks availability using separate functions (`is_torch_available()``, `is_tf_available()``, `is_flax_available()`) and appends corresponding framework abbreviations to the result.

**Key Implementation Details:**

* Checks availability of PyTorch, TensorFlow, and Flax using separate functions
* Appends abbreviated framework names ("pt", "tf", "flax") to the result if available

**Notable Dependencies/Requirements:**

* `is_torch_available()``, `is_tf_available()`, and `is_flax_available()`
* No external dependencies listed


key: chroma:document
string_value:
CODE:

def retrieve_model_classes(model_type: str, frameworks: Optional[List[str]] = None) -> Dict[str, List[str]]:
    """
    Retrieve the model classes associated to a given model.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            The frameworks to look for. Will default to `["pt", "tf", "flax"]`, passing a smaller list will restrict
            the classes returned.

    Returns:
        `Dict[str, List[str]]`: A dictionary with one key per framework and the list of model classes associated to
        that framework as values.
    """
    if frameworks is None:
        frameworks = get_default_frameworks()

    modules = {
        "pt": auto_module.modeling_auto if is_torch_available() else None,
        "tf": auto_module.modeling_tf_auto if is_tf_available() else None,
        "flax": auto_module.modeling_flax_auto if is_flax_available() else None,
    }

    model_classes = {}
    for framework in frameworks:
        new_model_classes = []
        if modules[framework] is None:
            raise ValueError(f"You selected {framework} in the frameworks, but it is not installed.")
        model_mappings = [attr for attr in dir(modules[framework]) if _re_model_mapping.search(attr) is not None]
        for model_mapping_name in model_mappings:
            model_mapping = getattr(modules[framework], model_mapping_name)
            if model_type in model_mapping:
                new_model_classes.append(model_mapping[model_type])

        if len(new_model_classes) > 0:
            # Remove duplicates
            model_classes[framework] = list(set(new_model_classes))



SUMMARY:
**Function Summary**

`retrieve_model_classes`: Retrieves the model classes associated with a given model type for specified frameworks.

* **Purpose**: Maps a given `model_type` to its corresponding classes in selected `frameworks`.
* **Input**:
	+ `model_type`: The desired model type (str)
	+ `frameworks`: Optional list of frameworks to look for (default: ["pt", "tf", "flax"])
* **Output**: A dictionary mapping each framework to a list of associated model classes.

**Key Implementation Details**

* Uses a dictionary to store mappings between frameworks and their corresponding model classes.
* Iterates over selected frameworks, checks for installed modules, and filters model classes based on the `model_type`.
* Removes duplicates from model class lists before returning the result.

**Notable Dependencies or Requirements**

* `auto_module` with extensions for "pt", "tf", and "flax" modeling architectures
* `is_torch_available`, `is_tf_available`, `is_flax_available` checks for module installations
* `_re_model_mapping.search` regular expression for matching model mapping attributes

**Method Signatures and Interfaces**

* No explicit interfaces, but uses `dir()` to access framework-specific modules.
* Uses `getattr()` to retrieve specific attributes from modules.


key: chroma:document
string_value:
CODE:

def retrieve_info_for_model(model_type, frameworks: Optional[List[str]] = None):
    """
    Retrieves all the information from a given model_type.

    Args:
        model_type (`str`): A valid model type (like "bert" or "gpt2")
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the info corresponding to the passed frameworks.

    Returns:
        `Dict`: A dictionary with the following keys:
        - **frameworks** (`List[str]`): The list of frameworks that back this model type.
        - **model_classes** (`Dict[str, List[str]]`): The model classes implemented for that model type.
        - **model_files** (`Dict[str, Union[Path, List[Path]]]`): The files associated with that model type.
        - **model_patterns** (`ModelPatterns`): The various patterns for the model.
    """
    if model_type not in auto_module.MODEL_NAMES_MAPPING:
        raise ValueError(f"{model_type} is not a valid model type.")

    model_name = auto_module.MODEL_NAMES_MAPPING[model_type]
    config_class = auto_module.configuration_auto.CONFIG_MAPPING_NAMES[model_type]
    if model_type in auto_module.tokenization_auto.TOKENIZER_MAPPING_NAMES:
        tokenizer_classes = auto_module.tokenization_auto.TOKENIZER_MAPPING_NAMES[model_type]
        tokenizer_class = tokenizer_classes[0] if tokenizer_classes[0] is not None else tokenizer_classes[1]
    else:
        tokenizer_class = None
    image_processor_classes = auto_module.image_processing_auto.IMAGE_PROCESSOR_MAPPING_NAMES.get(model_type, None)
    if isinstance(image_processor_classes, tuple):
        image_processor_class = image_processor_classes[0]  # we take the slow image processor class.
    else:
        image_processor_class = image_processor_classes

    feature_extractor_class = auto_module.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES.get(model_type, None)
    processor_class = auto_module.processing_auto.PROCESSOR_MAPPING_NAMES.get(model_type, None)

    model_files = get_model_files(model_type, frameworks=frameworks)
    model_camel_cased = config_class.replace("Config", "")

    available_frameworks = []
    for fname in model_files["model_files"]:
        if "modeling_tf" in str(fname):
            available_frameworks.append("tf")
        elif "modeling_flax" in str(fname):
            available_frameworks.append("flax")
        elif "modeling" in str(fname):
            available_frameworks.append("pt")

    if frameworks is None:
        frameworks = get_default_frameworks()

    frameworks = [f for f in frameworks if f in available_frameworks]

    model_classes = retrieve_model_classes(model_type, frameworks=frameworks)

    model_upper_cased = model_camel_cased.upper()
    model_patterns = ModelPatterns(
        model_name,
        checkpoint=find_base_model_checkpoint(model_type, model_files=model_files),
        model_type=model_type,
        model_camel_cased=model_camel_cased,
        model_lower_cased=model_files["module_name"],
        model_upper_cased=model_upper_cased,
        config_class=config_class,
        tokenizer_class=tokenizer_class,
        image_processor_class=image_processor_class,
        feature_extractor_class=feature_extractor_class,
        processor_class=processor_class,
    )

    return {
        "frameworks": frameworks,
        "model_classes": model_classes,
        "model_files": model_files,
        "model_patterns": model_patterns,


SUMMARY:
**Function Summary: `retrieve_info_for_model`**

 Retrieves information about a given model type, optionally filtering by specified frameworks.

### Main Purpose and Functionality

The function takes two parameters: `model_type` (a valid model name) and `frameworks` (an optional list of framework names). It returns a dictionary containing the following keys:
- `frameworks`: a list of supported frameworks for the model type.
- `model_classes`: a dictionary mapping models to their corresponding classes.
- `model_files`: a dictionary of files associated with the model type.
- `model_patterns`: an object representing various patterns for the model.

### Key Implementation Details

* It uses external modules and class names consistently, e.g., `auto_module.MODEL_NAMES_MAPPING`, `config_class`.
* The function filters frameworks based on available matching file names in `model_files`.
* The `tokenizer_class` is set as either a tuple-element or `None` depending on the framework type.
* The `image_processor_class` and `feature_extractor_class` are retrieved from their respective mappings for each model type.

### Notable Dependencies and Requirements

- External modules: `auto_module`
- Related classes/functions:
  - `auto_module.MODEL_NAMES_MAPPING`
  - `auto_module(configuration_auto.CONFIG_MAPPING_NAMES)`
  - `auto_module.tokenization_auto.TOKENIZER_MAPPING_NAMES`
  - `get_model_files` function
  - `get_default_frameworks` function

### Method Signatures and Interfaces

- The main function takes `model_type`, `frameworks`: `Optional[List[str]] = None`.
- Returns a dictionary with the specified keys.

Note that some external modules and functions (e.g., `find_base_model_checkpoint`) are not included in this summary, as their exact functionality is not essential to understanding the purpose and implementation of this function.


key: chroma:document
string_value:
CODE:

def clean_frameworks_in_init(
    init_file: Union[str, os.PathLike], frameworks: Optional[List[str]] = None, keep_processing: bool = True
):
    """
    Removes all the import lines that don't belong to a given list of frameworks or concern tokenizers/feature
    extractors/image processors/processors in an init.

    Args:
        init_file (`str` or `os.PathLike`): The path to the init to treat.
        frameworks (`List[str]`, *optional*):
           If passed, this will remove all imports that are subject to a framework not in frameworks
        keep_processing (`bool`, *optional*, defaults to `True`):
            Whether or not to keep the preprocessing (tokenizer, feature extractor, image processor, processor) imports
            in the init.
    """
    if frameworks is None:
        frameworks = get_default_frameworks()

    names = {"pt": "torch"}
    to_remove = [names.get(f, f) for f in ["pt", "tf", "flax"] if f not in frameworks]
    if not keep_processing:
        to_remove.extend(["sentencepiece", "tokenizers", "vision"])

    if len(to_remove) == 0:
        # Nothing to do
        return

    remove_pattern = "|".join(to_remove)
    re_conditional_imports = re.compile(rf"^\s*if not is_({remove_pattern})_available\(\):\s*$")
    re_try = re.compile(r"\s*try:")
    re_else = re.compile(r"\s*else:")
    re_is_xxx_available = re.compile(rf"is_({remove_pattern})_available")

    with open(init_file, "r", encoding="utf-8") as f:
        content = f.read()

    lines = content.split("\n")
    new_lines = []
    idx = 0
    while idx < len(lines):
        # Conditional imports in try-except-else blocks
        if (re_conditional_imports.search(lines[idx]) is not None) and (re_try.search(lines[idx - 1]) is not None):
            # Remove the preceding `try:`
            new_lines.pop()
            idx += 1
            # Iterate until `else:`
            while is_empty_line(lines[idx]) or re_else.search(lines[idx]) is None:
                idx += 1
            idx += 1
            indent = find_indent(lines[idx])
            while find_indent(lines[idx]) >= indent or is_empty_line(lines[idx]):
                idx += 1
        # Remove the import from utils
        elif re_is_xxx_available.search(lines[idx]) is not None:
            line = lines[idx]
            for framework in to_remove:
                line = line.replace(f", is_{framework}_available", "")
                line = line.replace(f"is_{framework}_available, ", "")
                line = line.replace(f"is_{framework}_available,", "")
                line = line.replace(f"is_{framework}_available", "")

            if len(line.strip()) > 0:
                new_lines.append(line)
            idx += 1
        # Otherwise we keep the line, except if it's a tokenizer import and we don't want to keep it.
        elif keep_processing or (
            re.search(r'^\s*"(tokenization|processing|feature_extraction|image_processing)', lines[idx]) is None
            and re.search(r"^\s*from .(tokenization|processing|feature_extraction|image_processing)", lines[idx])
            is None
        ):
            new_lines.append(lines[idx])
            idx += 1
        else:
            idx += 1

    with open(init_file, "w", encoding="utf-8") as f:


SUMMARY:
**Summary:**

This function `clean_frameworks_in_init` removes all import lines that don't belong to a given list of frameworks or concern tokenizers/feature extractors/image processors in an init.

**Key Details:**

* Removes imports for "pt", "tf", and "flax" frameworks if not provided in the list.
* Maintains preprocessing imports (torch) by default, unless `keep_processing` is False.
* Uses regular expressions to remove conditional imports and replaces import statements related to removed frameworks.

**Notable Dependencies/Requirements:**

* `os` module for handling file paths and encoding.
* `get_default_frameworks()` function (not shown in the code snippet).
* `re` module for regular expressions.

**Functionality:**

1. Reads an init file at the specified path.
2. Checks if a list of frameworks is provided; otherwise, uses default frameworks.
3. Filters out imports related to removed frameworks or tokenizers, unless `keep_processing` is set to False.
4. Writes the filtered content back to the original init file.


key: chroma:document
string_value:
CODE:

def add_model_to_main_init(
    old_model_patterns: ModelPatterns,
    new_model_patterns: ModelPatterns,
    frameworks: Optional[List[str]] = None,
    with_processing: bool = True,
):
    """
    Add a model to the main init of Transformers.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        frameworks (`List[str]`, *optional*):
            If specified, only the models implemented in those frameworks will be added.
        with_processsing (`bool`, *optional*, defaults to `True`):
            Whether the tokenizer/feature extractor/processor of the model should also be added to the init or not.
    """
    with open(TRANSFORMERS_PATH / "__init__.py", "r", encoding="utf-8") as f:
        content = f.read()

    lines = content.split("\n")
    idx = 0
    new_lines = []
    framework = None
    while idx < len(lines):
        new_framework = False
        if not is_empty_line(lines[idx]) and find_indent(lines[idx]) == 0:
            framework = None
        elif lines[idx].lstrip().startswith("if not is_torch_available"):
            framework = "pt"
            new_framework = True
        elif lines[idx].lstrip().startswith("if not is_tf_available"):
            framework = "tf"
            new_framework = True
        elif lines[idx].lstrip().startswith("if not is_flax_available"):
            framework = "flax"
            new_framework = True

        if new_framework:
            # For a new framework, we need to skip until the else: block to get where the imports are.
            while lines[idx].strip() != "else:":
                new_lines.append(lines[idx])
                idx += 1

        # Skip if we are in a framework not wanted.
        if framework is not None and frameworks is not None and framework not in frameworks:
            new_lines.append(lines[idx])
            idx += 1
        elif re.search(rf'models.{old_model_patterns.model_lower_cased}( |")', lines[idx]) is not None:
            block = [lines[idx]]
            indent = find_indent(lines[idx])
            idx += 1
            while find_indent(lines[idx]) > indent:
                block.append(lines[idx])
                idx += 1
            if lines[idx].strip() in [")", "]", "],"]:
                block.append(lines[idx])
                idx += 1
            block = "\n".join(block)
            new_lines.append(block)

            add_block = True
            if not with_processing:
                processing_classes = [
                    old_model_patterns.tokenizer_class,
                    old_model_patterns.image_processor_class,
                    old_model_patterns.feature_extractor_class,
                    old_model_patterns.processor_class,
                ]
                # Only keep the ones that are not None
                processing_classes = [c for c in processing_classes if c is not None]
                for processing_class in processing_classes:
                    block = block.replace(f' "{processing_class}",', "")
                    block = block.replace(f', "{processing_class}"', "")
                    block = block.replace(f" {processing_class},", "")
                    block = block.replace(f", {processing_class}", "")

                    if processing_class in block:
                        add_block = False
            if add_block:
                new_lines.append(replace_model_patterns(block, old_model_patterns, new_model_patterns)[0])
        else:
            new_lines.append(lines[idx])
            idx += 1

    with open(TRANSFORMERS_PATH / "__init__.py", "w", encoding="utf-8") as f:


SUMMARY:
**Function Summary:**

`add_model_to_main_init`
------------------------

Adds a model to the main init of Transformers. Replaces or removes sections based on framework specifications and processing flags.

### Functionality:

- Reads `__init__.py` file in the Transformers directory
- Filters lines by framework (if specified) and processing flag (`with_processing`)
- Updates `__init__.py` file with modified content

### Key Implementation Details:

- Checks for framework-specific imports and skips them if not wanted
- Updates tokenizer/feature extractor/processor block based on `with_processing` flag
- Replaces model patterns using the `add_block` function

### Dependencies/Requirements:

- Transformers library (`model_patterns`, `TRANSFORMERS_PATH`)
- Optional dependencies:
  - List of allowed frameworks (`frameworks`)
  - Processing flag (`with_processing`)
  - Tokenizer, feature extractor, and processor classes (implied by model patterns)

### Notable Interfaces/Methods:

- `find_indent`, `is_empty_line`, `find_model_pattern`
- `replace_model_patterns`


key: chroma:document
string_value:
CODE:

def insert_tokenizer_in_auto_module(old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns):
    """
    Add a tokenizer to the relevant mappings in the auto module.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
    """
    if old_model_patterns.tokenizer_class is None or new_model_patterns.tokenizer_class is None:
        return

    with open(TRANSFORMERS_PATH / "models" / "auto" / "tokenization_auto.py", "r", encoding="utf-8") as f:
        content = f.read()

    lines = content.split("\n")
    idx = 0
    # First we get to the TOKENIZER_MAPPING_NAMES block.
    while not lines[idx].startswith("    TOKENIZER_MAPPING_NAMES = OrderedDict("):
        idx += 1
    idx += 1

    # That block will end at this prompt:
    while not lines[idx].startswith("TOKENIZER_MAPPING = _LazyAutoMapping"):
        # Either all the tokenizer block is defined on one line, in which case, it ends with "),"
        if lines[idx].endswith(","):
            block = lines[idx]
        # Otherwise it takes several lines until we get to a "),"
        else:
            block = []
            while not lines[idx].startswith("            ),"):
                block.append(lines[idx])
                idx += 1
            block = "\n".join(block)
        idx += 1

        # If we find the model type and tokenizer class in that block, we have the old model tokenizer block
        if f'"{old_model_patterns.model_type}"' in block and old_model_patterns.tokenizer_class in block:
            break

    new_block = block.replace(old_model_patterns.model_type, new_model_patterns.model_type)
    new_block = new_block.replace(old_model_patterns.tokenizer_class, new_model_patterns.tokenizer_class)

    new_lines = lines[:idx] + [new_block] + lines[idx:]
    with open(TRANSFORMERS_PATH / "models" / "auto" / "tokenization_auto.py", "w", encoding="utf-8") as f:


SUMMARY:
**Summary of `insert_tokenizer_in_auto_module` function**

This function updates the tokenizer mappings in the `tokenization_auto.py` file within the auto module. The relevant block contains a mapping between model types and tokenizer classes.

**Key Implementation Details:**

- The function searches for a block containing model type and tokenizer class.
- It replaces the old model type and tokenizer class with the new values from the input patterns.
- The updated block is then written over the existing file, preserving all other lines up to the revised mapping.

**Notable Dependencies/Requirements:**

- `ModelPatterns` object with `model_type` and `tokenizer_class` attributes
- `TRANSFORMERS_PATH` directory path
- External dependencies included in `tokenization_auto.py` file

**Function Signature:**
```python
def insert_tokenizer_in_auto_module(old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns)
```
Note: The precise class and method names used exactly as they appear in the original code.


key: chroma:document
string_value:
CODE:

def add_model_to_auto_classes(
    old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns, model_classes: Dict[str, List[str]]
):
    """
    Add a model to the relevant mappings in the auto module.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        model_classes (`Dict[str, List[str]]`): A dictionary framework to list of model classes implemented.
    """
    for filename in AUTO_CLASSES_PATTERNS:
        # Extend patterns with all model classes if necessary
        new_patterns = []
        for pattern in AUTO_CLASSES_PATTERNS[filename]:
            if re.search("any_([a-z]*)_class", pattern) is not None:
                framework = re.search("any_([a-z]*)_class", pattern).groups()[0]
                if framework in model_classes:
                    new_patterns.extend(
                        [
                            pattern.replace("{" + f"any_{framework}_class" + "}", cls)
                            for cls in model_classes[framework]
                        ]
                    )
            elif "{config_class}" in pattern:
                new_patterns.append(pattern.replace("{config_class}", old_model_patterns.config_class))
            elif "{image_processor_class}" in pattern:
                if (
                    old_model_patterns.image_processor_class is not None
                    and new_model_patterns.image_processor_class is not None
                ):
                    new_patterns.append(
                        pattern.replace("{image_processor_class}", old_model_patterns.image_processor_class)
                    )
            elif "{feature_extractor_class}" in pattern:
                if (
                    old_model_patterns.feature_extractor_class is not None
                    and new_model_patterns.feature_extractor_class is not None
                ):
                    new_patterns.append(
                        pattern.replace("{feature_extractor_class}", old_model_patterns.feature_extractor_class)
                    )
            elif "{processor_class}" in pattern:
                if old_model_patterns.processor_class is not None and new_model_patterns.processor_class is not None:
                    new_patterns.append(pattern.replace("{processor_class}", old_model_patterns.processor_class))
            else:
                new_patterns.append(pattern)

        # Loop through all patterns.
        for pattern in new_patterns:
            full_name = TRANSFORMERS_PATH / "models" / "auto" / filename
            old_model_line = pattern
            new_model_line = pattern
            for attr in ["model_type", "model_name"]:
                old_model_line = old_model_line.replace("{" + attr + "}", getattr(old_model_patterns, attr))
                new_model_line = new_model_line.replace("{" + attr + "}", getattr(new_model_patterns, attr))
            new_model_line = new_model_line.replace(
                old_model_patterns.model_camel_cased, new_model_patterns.model_camel_cased
            )

            add_content_to_file(full_name, new_model_line, add_after=old_model_line)

    # Tokenizers require special handling


SUMMARY:
**Function Summary: `add_model_to_auto_classes`**

### Purpose and Functionality

The `add_model_to_auto_classes` function integrates two model patterns (`old_model_patterns` and `new_model_patterns`) into the auto module, updating relevant mappings. It uses pattern matching to replace placeholders with class names and values from model patterns.

**Key Implementation Details:**

* Extends `AUTO_CLASSES_PATTERNS` by replacing placeholders with actual class names.
* Handles template updates using regular expressions (`re.search`, `re.sub`).

**Notable Dependencies and Requirements:**

* `ModelPatterns`
* `Dict[str, List[str]]` for model classes
* External dependency: none (but the function relies on other defined functions like `add_content_to_file`)

### Function Signatures:

```python
def add_model_to_auto_classes(
    old_model_patterns: ModelPatterns, new_model_patterns: ModelPatterns, model_classes: Dict[str, List[str]]
):
```

This function is called when adding a new model to the auto module. It takes `old_model_patterns`, `new_model_patterns`, and `model_classes` as inputs.


key: chroma:document
string_value:
CODE:

def duplicate_doc_file(
    doc_file: Union[str, os.PathLike],
    old_model_patterns: ModelPatterns,
    new_model_patterns: ModelPatterns,
    dest_file: Optional[Union[str, os.PathLike]] = None,
    frameworks: Optional[List[str]] = None,
):
    """
    Duplicate a documentation file and adapts it for a new model.

    Args:
        module_file (`str` or `os.PathLike`): Path to the doc file to duplicate.
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        dest_file (`str` or `os.PathLike`, *optional*): Path to the new doc file.
            Will default to the a file named `{new_model_patterns.model_type}.md` in the same folder as `module_file`.
        frameworks (`List[str]`, *optional*):
            If passed, will only keep the model classes corresponding to this list of frameworks in the new doc file.
    """
    with open(doc_file, "r", encoding="utf-8") as f:
        content = f.read()

    content = re.sub(r"<!--\s*Copyright (\d+)\s", f"<!--Copyright {CURRENT_YEAR} ", content)
    if frameworks is None:
        frameworks = get_default_frameworks()
    if dest_file is None:
        dest_file = Path(doc_file).parent / f"{new_model_patterns.model_type}.md"

    # Parse the doc file in blocks. One block per section/header
    lines = content.split("\n")
    blocks = []
    current_block = []

    for line in lines:
        if line.startswith("#"):
            blocks.append("\n".join(current_block))
            current_block = [line]
        else:
            current_block.append(line)
    blocks.append("\n".join(current_block))

    new_blocks = []
    in_classes = False
    for block in blocks:
        # Copyright
        if not block.startswith("#"):
            new_blocks.append(block)
        # Main title
        elif re.search(r"^#\s+\S+", block) is not None:
            new_blocks.append(f"# {new_model_patterns.model_name}\n")
        # The config starts the part of the doc with the classes.
        elif not in_classes and old_model_patterns.config_class in block.split("\n")[0]:
            in_classes = True
            new_blocks.append(DOC_OVERVIEW_TEMPLATE.format(model_name=new_model_patterns.model_name))
            new_block, _ = replace_model_patterns(block, old_model_patterns, new_model_patterns)
            new_blocks.append(new_block)
        # In classes
        elif in_classes:
            in_classes = True
            block_title = block.split("\n")[0]
            block_class = re.search(r"^#+\s+(\S.*)$", block_title).groups()[0]
            new_block, _ = replace_model_patterns(block, old_model_patterns, new_model_patterns)

            if "Tokenizer" in block_class:
                # We only add the tokenizer if necessary
                if old_model_patterns.tokenizer_class != new_model_patterns.tokenizer_class:
                    new_blocks.append(new_block)
            elif "ImageProcessor" in block_class:
                # We only add the image processor if necessary
                if old_model_patterns.image_processor_class != new_model_patterns.image_processor_class:
                    new_blocks.append(new_block)
            elif "FeatureExtractor" in block_class:
                # We only add the feature extractor if necessary
                if old_model_patterns.feature_extractor_class != new_model_patterns.feature_extractor_class:
                    new_blocks.append(new_block)
            elif "Processor" in block_class:
                # We only add the processor if necessary
                if old_model_patterns.processor_class != new_model_patterns.processor_class:
                    new_blocks.append(new_block)
            elif block_class.startswith("Flax"):
                # We only add Flax models if in the selected frameworks
                if "flax" in frameworks:
                    new_blocks.append(new_block)
            elif block_class.startswith("TF"):
                # We only add TF models if in the selected frameworks
                if "tf" in frameworks:
                    new_blocks.append(new_block)
            elif len(block_class.split(" ")) == 1:
                # We only add PyTorch models if in the selected frameworks
                if "pt" in frameworks:
                    new_blocks.append(new_block)
            else:
                new_blocks.append(new_block)

    with open(dest_file, "w", encoding="utf-8") as f:


SUMMARY:
**Function Summary: `duplicate_doc_file`**

**Main Purpose:** Duplicate a documentation file and adapt it for a new model.

**Key Functionality:**

* Reads the input doc file (`doc_file`) and replaces patterns specified in `old_model_patterns` with those from `new_model_patterns`.
* Outputs the modified content to a new file (`dest_file`, default: `{new_model_patterns.model_type}.md`).
* Optionally filters out certain model classes based on the provided frameworks list.

**Key Implementation Details:**

* Patterns replacement using regular expressions.
* Content splitting and parsing into blocks (sections/header).
* Conditional formatting of doc blocks based on class names and framework selection.
* Writing modified content to a new file.

**Notable Dependencies/Requirements:**

* `ModelPatterns` ( custom class)
* `get_default_frameworks` function
* Regular expression module (`re`)
* Optional dependencies: `flax`, `tf`, `pt` for conditionally filtering out models.


key: chroma:document
string_value:
CODE:

def insert_model_in_doc_toc(old_model_patterns, new_model_patterns):
    """
    Insert the new model in the doc TOC, in the same section as the old model.

    Args:
        old_model_patterns (`ModelPatterns`): The patterns for the old model.
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
    """
    toc_file = REPO_PATH / "docs" / "source" / "en" / "_toctree.yml"
    with open(toc_file, "r", encoding="utf8") as f:
        content = yaml.safe_load(f)

    # Get to the model API doc
    api_idx = 0
    while content[api_idx]["title"] != "API":
        api_idx += 1
    api_doc = content[api_idx]["sections"]

    model_idx = 0
    while api_doc[model_idx]["title"] != "Models":
        model_idx += 1
    model_doc = api_doc[model_idx]["sections"]

    # Find the base model in the Toc
    old_model_type = old_model_patterns.model_type
    section_idx = 0
    while section_idx < len(model_doc):
        sections = [entry["local"] for entry in model_doc[section_idx]["sections"]]
        if f"model_doc/{old_model_type}" in sections:
            break

        section_idx += 1

    if section_idx == len(model_doc):
        old_model = old_model_patterns.model_name
        new_model = new_model_patterns.model_name
        print(f"Did not find {old_model} in the table of content, so you will need to add {new_model} manually.")
        return

    # Add the new model in the same toc
    toc_entry = {"local": f"model_doc/{new_model_patterns.model_type}", "title": new_model_patterns.model_name}
    model_doc[section_idx]["sections"].append(toc_entry)
    model_doc[section_idx]["sections"] = sorted(model_doc[section_idx]["sections"], key=lambda s: s["title"].lower())
    api_doc[model_idx]["sections"] = model_doc
    content[api_idx]["sections"] = api_doc

    with open(toc_file, "w", encoding="utf-8") as f:


SUMMARY:
**Function Summary**

```python
def insert_model_in_doc_toc(old_model_patterns, new_model_patterns)
```

**Purpose and Functionality**
Inserts the `new_model_patterns` into the doc TOC in the same section as the `old_model_patterns`, updating the table of contents (`toc_file`) accordingly.

**Key Implementation Details**

* Iterates through TOC sections to find the correct position for the new model.
* Updates the `model_doc` by adding a new entry with the `new_model_patterns.model_name`.
* Re-constructs the updated content and writes it back to `toc_file`.

**Notable Dependencies and Requirements**

* `REPO_PATH`: Path to repository root directory.
* `yaml`: YAML parsing library for loading/treating TOC file content.
* `ModelPatterns`: Class containing patterns for old model and new model (not shown in this snippet).

**External Dependencies**

* None mentioned; no external libraries besides YAML are used.


key: chroma:document
string_value:
CODE:

def create_new_model_like(
    model_type: str,
    new_model_patterns: ModelPatterns,
    add_copied_from: bool = True,
    frameworks: Optional[List[str]] = None,
    old_checkpoint: Optional[str] = None,
):
    """
    Creates a new model module like a given model of the Transformers library.

    Args:
        model_type (`str`): The model type to duplicate (like "bert" or "gpt2")
        new_model_patterns (`ModelPatterns`): The patterns for the new model.
        add_copied_from (`bool`, *optional*, defaults to `True`):
            Whether or not to add "Copied from" statements to all classes in the new model modeling files.
        frameworks (`List[str]`, *optional*):
            If passed, will limit the duplicate to the frameworks specified.
        old_checkpoint (`str`, *optional*):
            The name of the base checkpoint for the old model. Should be passed along when it can't be automatically
            recovered from the `model_type`.
    """
    # Retrieve all the old model info.
    model_info = retrieve_info_for_model(model_type, frameworks=frameworks)
    model_files = model_info["model_files"]
    old_model_patterns = model_info["model_patterns"]
    if old_checkpoint is not None:
        old_model_patterns.checkpoint = old_checkpoint
    if len(old_model_patterns.checkpoint) == 0:
        raise ValueError(
            "The old model checkpoint could not be recovered from the model type. Please pass it to the "
            "`old_checkpoint` argument."
        )

    keep_old_processing = True
    for processing_attr in ["image_processor_class", "feature_extractor_class", "processor_class", "tokenizer_class"]:
        if getattr(old_model_patterns, processing_attr) != getattr(new_model_patterns, processing_attr):
            keep_old_processing = False

    model_classes = model_info["model_classes"]

    # 1. We create the module for our new model.
    old_module_name = model_files["module_name"]
    module_folder = TRANSFORMERS_PATH / "models" / new_model_patterns.model_lower_cased
    os.makedirs(module_folder, exist_ok=True)

    files_to_adapt = model_files["model_files"]
    if keep_old_processing:
        files_to_adapt = [
            f
            for f in files_to_adapt
            if "tokenization" not in str(f)
            and "processing" not in str(f)
            and "feature_extraction" not in str(f)
            and "image_processing" not in str(f)
        ]

    os.makedirs(module_folder, exist_ok=True)
    for module_file in files_to_adapt:
        new_module_name = module_file.name.replace(
            old_model_patterns.model_lower_cased, new_model_patterns.model_lower_cased
        )
        dest_file = module_folder / new_module_name
        duplicate_module(
            module_file,
            old_model_patterns,
            new_model_patterns,
            dest_file=dest_file,
            add_copied_from=add_copied_from and "modeling" in new_module_name,
        )

    clean_frameworks_in_init(
        module_folder / "__init__.py", frameworks=frameworks, keep_processing=not keep_old_processing
    )

    # 2. We add our new model to the models init and the main init
    add_content_to_file(
        TRANSFORMERS_PATH / "models" / "__init__.py",
        f"    {new_model_patterns.model_lower_cased},",
        add_after=f"    {old_module_name},",
        exact_match=True,
    )
    add_model_to_main_init(
        old_model_patterns, new_model_patterns, frameworks=frameworks, with_processing=not keep_old_processing
    )

    # 3. Add test files
    files_to_adapt = model_files["test_files"]
    if keep_old_processing:
        files_to_adapt = [
            f
            for f in files_to_adapt
            if "tokenization" not in str(f)
            and "processor" not in str(f)
            and "feature_extraction" not in str(f)
            and "image_processing" not in str(f)
        ]

    def disable_fx_test(filename: Path) -> bool:
        with open(filename) as fp:
            content = fp.read()
        new_content = re.sub(r"fx_compatible\s*=\s*True", "fx_compatible = False", content)
        with open(filename, "w") as fp:
            fp.write(new_content)
        return content != new_content

    disabled_fx_test = False

    tests_folder = REPO_PATH / "tests" / "models" / new_model_patterns.model_lower_cased
    os.makedirs(tests_folder, exist_ok=True)
    with open(tests_folder / "__init__.py", "w"):
        pass

    for test_file in files_to_adapt:
        new_test_file_name = test_file.name.replace(
            old_model_patterns.model_lower_cased, new_model_patterns.model_lower_cased
        )
        dest_file = test_file.parent.parent / new_model_patterns.model_lower_cased / new_test_file_name
        duplicate_module(
            test_file,
            old_model_patterns,
            new_model_patterns,
            dest_file=dest_file,
            add_copied_from=False,
            attrs_to_remove=["pipeline_model_mapping", "is_pipeline_test_to_skip"],
        )
        disabled_fx_test = disabled_fx_test | disable_fx_test(dest_file)

    if disabled_fx_test:
        print(
            "The tests for symbolic tracing with torch.fx were disabled, you can add those once symbolic tracing works"
            " for your new model."
        )

    # 4. Add model to auto classes
    add_model_to_auto_classes(old_model_patterns, new_model_patterns, model_classes)

    # 5. Add doc file
    doc_file = REPO_PATH / "docs" / "source" / "en" / "model_doc" / f"{old_model_patterns.model_type}.md"
    duplicate_doc_file(doc_file, old_model_patterns, new_model_patterns, frameworks=frameworks)
    insert_model_in_doc_toc(old_model_patterns, new_model_patterns)

    # 6. Warn the user for duplicate patterns
    if old_model_patterns.model_type == old_model_patterns.checkpoint:
        print(
            "The model you picked has the same name for the model type and the checkpoint name "
            f"({old_model_patterns.model_type}). As a result, it's possible some places where the new checkpoint "
            f"should be, you have {new_model_patterns.model_type} instead. You should search for all instances of "
            f"{new_model_patterns.model_type} in the new files and check they're not badly used as checkpoints."
        )
    elif old_model_patterns.model_lower_cased == old_model_patterns.checkpoint:
        print(
            "The model you picked has the same name for the model type and the checkpoint name "
            f"({old_model_patterns.model_lower_cased}). As a result, it's possible some places where the new "
            f"checkpoint should be, you have {new_model_patterns.model_lower_cased} instead. You should search for "
            f"all instances of {new_model_patterns.model_lower_cased} in the new files and check they're not badly "
            "used as checkpoints."
        )
    if (
        old_model_patterns.model_type == old_model_patterns.model_lower_cased
        and new_model_patterns.model_type != new_model_patterns.model_lower_cased
    ):
        print(
            "The model you picked has the same name for the model type and the lowercased model name "
            f"({old_model_patterns.model_lower_cased}). As a result, it's possible some places where the new "
            f"model type should be, you have {new_model_patterns.model_lower_cased} instead. You should search for "
            f"all instances of {new_model_patterns.model_lower_cased} in the new files and check they're not badly "
            "used as the model type."
        )

    if not keep_old_processing and old_model_patterns.tokenizer_class is not None:
        print(
            "The constants at the start of the new tokenizer file created needs to be manually fixed. If your new "
            "model has a tokenizer fast, you will also need to manually add the converter in the "
            "`SLOW_TO_FAST_CONVERTERS` constant of `convert_slow_tokenizer.py`."


SUMMARY:
**Summary**

This function, `create_new_model_like`, generates a new model similar to an existing Transformer-based model. It:

1.  Duplicates files and classes from the old model based on specified patterns.
2.  Updates framework configurations and removes framework dependencies where possible.
3.  Adds the new model to the main init file and models init file.
4.  Copies test files and tests their compatibility with torch fx.
5.  Creates an auto class for the new model.
6.  Inserts a doc file in the documentation source directory.

**Functionality Breakdown**

The function can be categorized into six main tasks:

*   **File Duplications**: Duplicate specific files from the old model to create corresponding files for the new model, including tokenizer classes with updated constants.
*   **Framework Configuration Updates**: Update frameworks and remove unnecessary framework dependencies where possible.
*   **Auto Class Creation**: Create an auto class for the new model using data from both files.
*   **Doc File Insertion**: Copy a doc file from a source directory to the repository's documentation source directory.


key: chroma:document
string_value:
CODE:

def add_new_model_like_command_factory(args: Namespace):


SUMMARY:
**Summary**

Creates an instance of `AddNewModelLikeCommand` with configuration settings from the provided namespace.

**Key Details**

- Uses `Args.Namespace` for type hinting
- Conveys functionality through `config_file` and `path_to_repo` arguments, which are passed to the constructor of `AddNewModelLikeCommand`

**Notable Dependencies/Requirements**

- `Namespace` from an external library (e.g., `argparse`)
- `AddNewModelLikeCommand` class implementation not shown in this code snippet


key: chroma:document
string_value:
CODE:

class AddNewModelLikeCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        add_new_model_like_parser = parser.add_parser("add-new-model-like")
        add_new_model_like_parser.add_argument(
            "--config_file", type=str, help="A file with all the information for this model creation."
        )
        add_new_model_like_parser.add_argument(
            "--path_to_repo", type=str, help="When not using an editable install, the path to the Transformers repo."
        )
        add_new_model_like_parser.set_defaults(func=add_new_model_like_command_factory)

    def __init__(self, config_file=None, path_to_repo=None, *args):
        if config_file is not None:
            with open(config_file, "r", encoding="utf-8") as f:
                config = json.load(f)
            self.old_model_type = config["old_model_type"]
            self.model_patterns = ModelPatterns(**config["new_model_patterns"])
            self.add_copied_from = config.get("add_copied_from", True)
            self.frameworks = config.get("frameworks", get_default_frameworks())
            self.old_checkpoint = config.get("old_checkpoint", None)
        else:
            (
                self.old_model_type,
                self.model_patterns,
                self.add_copied_from,
                self.frameworks,
                self.old_checkpoint,
            ) = get_user_input()

        self.path_to_repo = path_to_repo

    def run(self):
        if self.path_to_repo is not None:
            # Adapt constants
            global TRANSFORMERS_PATH
            global REPO_PATH

            REPO_PATH = Path(self.path_to_repo)
            TRANSFORMERS_PATH = REPO_PATH / "src" / "transformers"

        create_new_model_like(
            model_type=self.old_model_type,
            new_model_patterns=self.model_patterns,
            add_copied_from=self.add_copied_from,
            frameworks=self.frameworks,
            old_checkpoint=self.old_checkpoint,


SUMMARY:
**Class Summary: AddNewModelLikeCommand**

The `AddNewModelLikeCommand` class is a subclass of `BaseTransformersCLICommand`. It handles the "add-new-model-like" subcommand in a command-line interface (CLI).

**Primary Purpose and Functionality**

This class provides a CLI entry point for creating new models, similar to existing models in the Transformers framework. The primary function is to initialize and configure model creation parameters from either a configuration file or user input.

**Key Implementation Details**

* The `register_subcommand` method registers the "add-new-model-like" subcommand with the CLI parser.
* The `__init__` method loads configuration data from a file, if provided, and sets default values if no configuration is found. It also retrieves framework dependencies automatically.
* The `run` method adapts global constants and creates a new model using the configured parameters.

**Notable Dependencies or Requirements**

* The `ModelPatterns` class is required to parse patterns for new model creation.
* The `get_user_input` function returns default values if no configuration file is provided.
* External dependencies are:
	+ `json` for loading configuration data from files (.config).
	+ `Path` and ` glob` (implicit, as part of the Python standard library) for managing repository paths.

**Interfaces and Inherited Behavior**

This class inherits behavior from its parent class, `BaseTransformersCLICommand`, which likely handles common CLI tasks such as argument parsing and error handling.


key: chroma:document
string_value:
CODE:

def get_user_field(
    question: str,
    default_value: Optional[str] = None,
    is_valid_answer: Optional[Callable] = None,
    convert_to: Optional[Callable] = None,
    fallback_message: Optional[str] = None,
) -> Any:
    """
    A utility function that asks a question to the user to get an answer, potentially looping until it gets a valid
    answer.

    Args:
        question (`str`): The question to ask the user.
        default_value (`str`, *optional*): A potential default value that will be used when the answer is empty.
        is_valid_answer (`Callable`, *optional*):
            If set, the question will be asked until this function returns `True` on the provided answer.
        convert_to (`Callable`, *optional*):
            If set, the answer will be passed to this function. If this function raises an error on the procided
            answer, the question will be asked again.
        fallback_message (`str`, *optional*):
            A message that will be displayed each time the question is asked again to the user.

    Returns:
        `Any`: The answer provided by the user (or the default), passed through the potential conversion function.
    """
    if not question.endswith(" "):
        question = question + " "
    if default_value is not None:
        question = f"{question} [{default_value}] "

    valid_answer = False
    while not valid_answer:
        answer = input(question)
        if default_value is not None and len(answer) == 0:
            answer = default_value
        if is_valid_answer is not None:
            valid_answer = is_valid_answer(answer)
        elif convert_to is not None:
            try:
                answer = convert_to(answer)
                valid_answer = True
            except Exception:
                valid_answer = False
        else:
            valid_answer = True

        if not valid_answer:
            print(fallback_message)



SUMMARY:
**Summary: `get_user_field` Function**

### Purpose and Functionality

The `get_user_field` function is a utility tool that prompts a user to provide an input field, potentially looping until a valid response is obtained.

### Key Implementation Details

* Takes in a question string and optional parameters for:
	+ `default_value`: default value to use if answer is empty
	+ `is_valid_answer`: callback function to validate answer
	+ `convert_to`: callback function to convert input into desired format
	+ `fallback_message`: message displayed when looping back to user with the same question
* Uses a while loop to continuously ask the user for input until a valid response is provided
* Handles potential errors by skipping to the next iteration or displaying the fallback message

### Dependencies and Requirements

* `Callable` type hint for callback functions (`is_valid_answer`, `convert_to`)
* No external dependencies noted.


key: chroma:document
string_value:
CODE:

def convert_to_bool(x: str) -> bool:
    """
    Converts a string to a bool.
    """
    if x.lower() in ["1", "y", "yes", "true"]:
        return True
    if x.lower() in ["0", "n", "no", "false"]:
        return False


SUMMARY:
**Function Summary**

`convert_to_bool(x: str) -> bool`

Converts a string input `x` to a boolean value.

**Implementation Details**

- Uses case-insensitive comparison (`x.lower()`) for matching string values.
- Provides a list of approved values ("1", "y", "yes", "true") that are converted to `True`.
- Provides a list of rejected values ("0", "n", "no", "false") that result in an exception.

**Notable Dependencies or Requirements**

- None
- No external dependencies
- Follows standard Python convention for boolean conversion.


key: chroma:document
string_value:
CODE:

def get_user_input():
    """
    Ask the user for the necessary inputs to add the new model.
    """
    model_types = list(auto_module.configuration_auto.MODEL_NAMES_MAPPING.keys())

    # Get old model type
    valid_model_type = False
    while not valid_model_type:
        old_model_type = input(
            "What is the model you would like to duplicate? Please provide the lowercase `model_type` (e.g. roberta): "
        )
        if old_model_type in model_types:
            valid_model_type = True
        else:
            print(f"{old_model_type} is not a valid model type.")
            near_choices = difflib.get_close_matches(old_model_type, model_types)
            if len(near_choices) >= 1:
                if len(near_choices) > 1:
                    near_choices = " or ".join(near_choices)
                print(f"Did you mean {near_choices}?")

    old_model_info = retrieve_info_for_model(old_model_type)
    old_tokenizer_class = old_model_info["model_patterns"].tokenizer_class
    old_image_processor_class = old_model_info["model_patterns"].image_processor_class
    old_feature_extractor_class = old_model_info["model_patterns"].feature_extractor_class
    old_processor_class = old_model_info["model_patterns"].processor_class
    old_frameworks = old_model_info["frameworks"]

    old_checkpoint = None
    if len(old_model_info["model_patterns"].checkpoint) == 0:
        old_checkpoint = get_user_field(
            "We couldn't find the name of the base checkpoint for that model, please enter it here."
        )

    model_name = get_user_field(
        "What is the name (with no special casing) for your new model in the paper (e.g. RoBERTa)? "
    )
    default_patterns = ModelPatterns(model_name, model_name)

    model_type = get_user_field(
        "What identifier would you like to use for the `model_type` of this model? ",
        default_value=default_patterns.model_type,
    )
    model_lower_cased = get_user_field(
        "What lowercase name would you like to use for the module (folder) of this model? ",
        default_value=default_patterns.model_lower_cased,
    )
    model_camel_cased = get_user_field(
        "What prefix (camel-cased) would you like to use for the model classes of this model (e.g. Roberta)? ",
        default_value=default_patterns.model_camel_cased,
    )
    model_upper_cased = get_user_field(
        "What prefix (upper-cased) would you like to use for the constants relative to this model? ",
        default_value=default_patterns.model_upper_cased,
    )
    config_class = get_user_field(
        "What will be the name of the config class for this model? ", default_value=f"{model_camel_cased}Config"
    )
    checkpoint = get_user_field(
        "Please give a checkpoint identifier (on the model Hub) for this new model (e.g. facebook/FacebookAI/roberta-base): "
    )

    old_processing_classes = [
        c if not isinstance(c, tuple) else c[0]
        for c in [old_image_processor_class, old_feature_extractor_class, old_tokenizer_class, old_processor_class]
        if c is not None
    ]
    old_processing_classes = ", ".join(old_processing_classes)
    keep_processing = get_user_field(
        f"Will your new model use the same processing class as {old_model_type} ({old_processing_classes}) (yes/no)? ",
        convert_to=convert_to_bool,
        fallback_message="Please answer yes/no, y/n, true/false or 1/0. ",
    )
    if keep_processing:
        image_processor_class = old_image_processor_class
        feature_extractor_class = old_feature_extractor_class
        processor_class = old_processor_class
        tokenizer_class = old_tokenizer_class
    else:
        if old_tokenizer_class is not None:
            tokenizer_class = get_user_field(
                "What will be the name of the tokenizer class for this model? ",
                default_value=f"{model_camel_cased}Tokenizer",
            )
        else:
            tokenizer_class = None
        if old_image_processor_class is not None:
            image_processor_class = get_user_field(
                "What will be the name of the image processor class for this model? ",
                default_value=f"{model_camel_cased}ImageProcessor",
            )
        else:
            image_processor_class = None
        if old_feature_extractor_class is not None:
            feature_extractor_class = get_user_field(
                "What will be the name of the feature extractor class for this model? ",
                default_value=f"{model_camel_cased}FeatureExtractor",
            )
        else:
            feature_extractor_class = None
        if old_processor_class is not None:
            processor_class = get_user_field(
                "What will be the name of the processor class for this model? ",
                default_value=f"{model_camel_cased}Processor",
            )
        else:
            processor_class = None

    model_patterns = ModelPatterns(
        model_name,
        checkpoint,
        model_type=model_type,
        model_lower_cased=model_lower_cased,
        model_camel_cased=model_camel_cased,
        model_upper_cased=model_upper_cased,
        config_class=config_class,
        tokenizer_class=tokenizer_class,
        image_processor_class=image_processor_class,
        feature_extractor_class=feature_extractor_class,
        processor_class=processor_class,
    )

    add_copied_from = get_user_field(
        "Should we add # Copied from statements when creating the new modeling file (yes/no)? ",
        convert_to=convert_to_bool,
        default_value="yes",
        fallback_message="Please answer yes/no, y/n, true/false or 1/0.",
    )

    all_frameworks = get_user_field(
        "Should we add a version of your new model in all the frameworks implemented by"
        f" {old_model_type} ({old_frameworks}) (yes/no)? ",
        convert_to=convert_to_bool,
        default_value="yes",
        fallback_message="Please answer yes/no, y/n, true/false or 1/0.",
    )
    if all_frameworks:
        frameworks = None
    else:
        frameworks = get_user_field(
            "Please enter the list of framworks you want (pt, tf, flax) separated by spaces",
            is_valid_answer=lambda x: all(p in ["pt", "tf", "flax"] for p in x.split(" ")),
        )
        frameworks = list(set(frameworks.split(" ")))



SUMMARY:
**Function Summary: `get_user_input`**

### Purpose and Functionality

This function asks the user for input data to create a new model. It collects user input for various parameters, such as model name, identifier, checkpoint, frameworks, and additional fields.

### Key Implementation Details

*   Asks the user to select a valid old model type using `difflib.get_close_matches`.
*   Retrieves information for the selected old model type.
*   Asks the user to input fields such as:
    *   Model name
    *   Identifier (model type)
    *   Lowercase identifier (module folder)
    *   Camel-cased identifier (model class prefix)
    *   Upper-cased identifier (constant prefix)
    *   Config class name
    *   Checkpoint identifier
*   Determines which processing classes (image, feature extractor, tokenizer, processor) to use based on user input.
*   Asks the user if additional fields should be added:
    *   Copied from statements when creating a new modeling file
    *   Version of the model for each framework implemented by the old model type

### Notable Dependencies and Requirements

*   `difflib` for close matching functionality
*   `auto_module.configuration_auto.MODEL_NAMES_MAPPING` for old model types
*   `retrieve_info_for_model` function to retrieve information for the selected old model type
*   `gets_user_field` functions to collect user input for various fields


key: chroma:document
string_value:
CODE:

def download_command_factory(args):


SUMMARY:
**Summary:**

The `download_command_factory` function creates and returns an instance of `DownloadCommand`, which is used to initiate the download process. It takes four arguments:

- `model`
- `cache_dir`
- `force`
- `trust_remote_code`

 **Main Purpose:** Creates a factory for DownloadCommand instances.

**Key Implementation Details:**

* Initializes DownloadCommand with specific parameters (`model`, `cacheDir`, `force`, and `trustRemoteCode`).

**Notable Dependencies/Requirements:**

* The function relies on the existence of `DownloadCommand`, which is not shown in this snippet.
* No external dependencies are listed.


key: chroma:document
string_value:
CODE:

class DownloadCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        download_parser = parser.add_parser("download")
        download_parser.add_argument(
            "--cache-dir", type=str, default=None, help="Path to location to store the models"
        )
        download_parser.add_argument(
            "--force", action="store_true", help="Force the model to be download even if already in cache-dir"
        )
        download_parser.add_argument(
            "--trust-remote-code",
            action="store_true",
            help="Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you've reviewed the code as it will execute on your local machine",
        )
        download_parser.add_argument("model", type=str, help="Name of the model to download")
        download_parser.set_defaults(func=download_command_factory)

    def __init__(self, model: str, cache: str, force: bool, trust_remote_code: bool):
        self._model = model
        self._cache = cache
        self._force = force
        self._trust_remote_code = trust_remote_code

    def run(self):
        from ..models.auto import AutoModel, AutoTokenizer

        AutoModel.from_pretrained(
            self._model, cache_dir=self._cache, force_download=self._force, trust_remote_code=self._trust_remote_code
        )
        AutoTokenizer.from_pretrained(
            self._model, cache_dir=self._cache, force_download=self._force, trust_remote_code=self._trust_remote_code


SUMMARY:
**DownloadCommand Class Summary**

### Purpose and Functionality

The `DownloadCommand` class is a CLI command that allows users to download pre-trained models from the Hub. It provides options for customizing model storage and remote code execution.

### Key Implementation Details

*   The `download_command_factory` function is set as the default action for the "download" subcommand, which retrieves the AutoModel and AutoTokenizer instances.
*   The class uses `BaseTransformersCLICommand` as its base class to extend CLIParser functionality.

    ### Notable Dependencies or Requirements
    *   `ArgumentParser`
    *   `BaseTransformersCLICommand`

    The class inherits from `BaseTransformersCLICommand`, and utilizes the `ArgumentParser` to define the command-line interface for the "download" subcommand.


key: chroma:document
string_value:
CODE:

def train_command_factory(args: Namespace):
    """
    Factory function used to instantiate training command from provided command line arguments.

    Returns: TrainCommand
    """


SUMMARY:
**Summary**

`train_command_factory`: Factory function to create `TrainCommand` instance from provided argument namespace (`args`) with primary purpose of instantiating training commands from command line arguments. 

### Key Details:

* Returns an instance of `TrainCommand`
* Uses `args` namespace as input

### Notable Dependencies:

* None


key: chroma:document
string_value:
CODE:

class TrainCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli

        Args:
            parser: Root parser to register command-specific arguments
        """
        train_parser = parser.add_parser("train", help="CLI tool to train a model on a task.")

        train_parser.add_argument(
            "--train_data",
            type=str,
            required=True,
            help="path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences.",
        )
        train_parser.add_argument(
            "--column_label", type=int, default=0, help="Column of the dataset csv file with example labels."
        )
        train_parser.add_argument(
            "--column_text", type=int, default=1, help="Column of the dataset csv file with example texts."
        )
        train_parser.add_argument(
            "--column_id", type=int, default=2, help="Column of the dataset csv file with example ids."
        )
        train_parser.add_argument(
            "--skip_first_row", action="store_true", help="Skip the first row of the csv file (headers)."
        )

        train_parser.add_argument("--validation_data", type=str, default="", help="path to validation dataset.")
        train_parser.add_argument(
            "--validation_split",
            type=float,
            default=0.1,
            help="if validation dataset is not provided, fraction of train dataset to use as validation dataset.",
        )

        train_parser.add_argument("--output", type=str, default="./", help="path to saved the trained model.")

        train_parser.add_argument(
            "--task", type=str, default="text_classification", help="Task to train the model on."
        )
        train_parser.add_argument(
            "--model", type=str, default="google-bert/bert-base-uncased", help="Model's name or path to stored model."
        )
        train_parser.add_argument("--train_batch_size", type=int, default=32, help="Batch size for training.")
        train_parser.add_argument("--valid_batch_size", type=int, default=64, help="Batch size for validation.")
        train_parser.add_argument("--learning_rate", type=float, default=3e-5, help="Learning rate.")
        train_parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon for Adam optimizer.")
        train_parser.set_defaults(func=train_command_factory)

    def __init__(self, args: Namespace):
        self.logger = logging.get_logger("transformers-cli/training")

        self.framework = "tf" if is_tf_available() else "torch"

        os.makedirs(args.output, exist_ok=True)
        self.output = args.output

        self.column_label = args.column_label
        self.column_text = args.column_text
        self.column_id = args.column_id

        self.logger.info(f"Loading {args.task} pipeline for {args.model}")
        if args.task == "text_classification":
            self.pipeline = TextClassificationPipeline.from_pretrained(args.model)
        elif args.task == "token_classification":
            raise NotImplementedError
        elif args.task == "question_answering":
            raise NotImplementedError

        self.logger.info(f"Loading dataset from {args.train_data}")
        self.train_dataset = Processor.create_from_csv(
            args.train_data,
            column_label=args.column_label,
            column_text=args.column_text,
            column_id=args.column_id,
            skip_first_row=args.skip_first_row,
        )
        self.valid_dataset = None
        if args.validation_data:
            self.logger.info(f"Loading validation dataset from {args.validation_data}")
            self.valid_dataset = Processor.create_from_csv(
                args.validation_data,
                column_label=args.column_label,
                column_text=args.column_text,
                column_id=args.column_id,
                skip_first_row=args.skip_first_row,
            )

        self.validation_split = args.validation_split
        self.train_batch_size = args.train_batch_size
        self.valid_batch_size = args.valid_batch_size
        self.learning_rate = args.learning_rate
        self.adam_epsilon = args.adam_epsilon

    def run(self):
        if self.framework == "tf":
            return self.run_tf()
        return self.run_torch()

    def run_torch(self):
        raise NotImplementedError

    def run_tf(self):
        self.pipeline.fit(
            self.train_dataset,
            validation_data=self.valid_dataset,
            validation_split=self.validation_split,
            learning_rate=self.learning_rate,
            adam_epsilon=self.adam_epsilon,
            train_batch_size=self.train_batch_size,
            valid_batch_size=self.valid_batch_size,
        )

        # Save trained pipeline


SUMMARY:
**TrainCommand Class Summary**

### Purpose and Functionality

The `TrainCommand` class is a CLI tool for training a machine learning model using a dataset. It takes in various arguments, including the model architecture (`google-bert/bert-base-uncased` or custom), the task type (`text_classification`, `token_classification`, or `question_answering`), and hyperparameters such as batch size, learning rate, and Adam optimizer epsilon.

### Key Implementation Details

* Registers command-line arguments using `ArgumentParser`
* Creates `Pipeline` instance for each model architecture
* Loads dataset from CSV file(s) and processes it into a `Processor` object
* Trains the `Pipeline` on the processed dataset with validation data (if provided)
* Saves the trained pipeline to a directory

### Notable Dependencies and Requirements

* `BaseTransformersCLICommand` from `transformers-cli`
* `ArgumentParser` from `argparse`
* `Logging` for logging
* `Processor` for loading and processing datasets
* `TextClassificationPipeline` or other custom pipelines for specific tasks

### Class Members

* `logger`: instance of Logger class
* `framework`: either "tf" (TensorFlow) or "torch"
* `output`: path to saved trained model directory
* `column_label`, `column_text`, and `column_id`: CSV column indices
* `task`: task type (`text_classification`, `token_classification`, or `question_answering`)
* `model`: model architecture or path to stored model
* `validation_data` and `validation_split`: validation dataset path and split fraction for training


key: chroma:document
string_value:
CODE:

class BaseTransformersCLICommand(ABC):
    @staticmethod
    @abstractmethod
    def register_subcommand(parser: ArgumentParser):
        raise NotImplementedError()

    @abstractmethod
    def run(self):


SUMMARY:
**Class Summary**

`BaseTransformersCLICommand`: Abstract Base Class for Command-Line Interface Transformers.

**Primary Purpose**: Define the base structure for command-line interface transformers in a CLI system, requiring registration of subcommands and implementation of the run method.

**Key Implementation Details**:

*   **`register_subcommand` Method**: Register a subcommand with an `ArgumentParser`.
    *   Method Signature: `@staticmethod def register_subcommand(parser: ArgumentParser)`
*   **`run` Method**: Perform the actual transformation or execution.
    *   Method Signature: `@abstractmethod def run(self):`

**Notable Dependencies and Requirements**:

*   Uses the `ArgumentParser` from the `argparse` library for subcommand registration.
*   Requires inheritance from `ABC` (Abstract Base Class) to define abstract methods.


key: chroma:document
string_value:
CODE:

def convert_command_factory(args: Namespace):
    """
    Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint.

    Returns: ServeCommand
    """
    return ConvertCommand(
        args.model_type, args.tf_checkpoint, args.pytorch_dump_output, args.config, args.finetuning_task_name


SUMMARY:
**Function Summary**

### Purpose and Functionality

The `convert_command_factory` function is a factory function that creates and returns an instance of the `ConvertCommand` class. Its primary purpose is to convert a TF 1.0 checkpoint to a PyTorch checkpoint.

### Key Implementation Details

* The function takes a `Namespace` object named `args` as input.
* It uses this input `args` to create a new instance of the `ConvertCommand` class, passing in several attributes (model type, TensorFlow checkpoint, PyTorch dump output, configuration, and fine-tuning task name).

### Notable Dependencies or Requirements

* The function relies on the `Namespace` object from the `argparse` library.
* It requires an instance of `ConvertCommand`.

**Summary Output**

`convert_command_factory`: Factory function to convert TF 1.0 checkpoint to PyTorch checkpoint, utilizing `ConvertCommand` class with input attributes (model type, TensorFlow checkpoint, etc.).


key: chroma:document
string_value:
CODE:

class ConvertCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli

        Args:
            parser: Root parser to register command-specific arguments
        """
        train_parser = parser.add_parser(
            "convert",
            help="CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.",
        )
        train_parser.add_argument("--model_type", type=str, required=True, help="Model's type.")
        train_parser.add_argument(
            "--tf_checkpoint", type=str, required=True, help="TensorFlow checkpoint path or folder."
        )
        train_parser.add_argument(
            "--pytorch_dump_output", type=str, required=True, help="Path to the PyTorch saved model output."
        )
        train_parser.add_argument("--config", type=str, default="", help="Configuration file path or folder.")
        train_parser.add_argument(
            "--finetuning_task_name",
            type=str,
            default=None,
            help="Optional fine-tuning task name if the TF model was a finetuned model.",
        )
        train_parser.set_defaults(func=convert_command_factory)

    def __init__(
        self,
        model_type: str,
        tf_checkpoint: str,
        pytorch_dump_output: str,
        config: str,
        finetuning_task_name: str,
        *args,
    ):
        self._logger = logging.get_logger("transformers-cli/converting")

        self._logger.info(f"Loading model {model_type}")
        self._model_type = model_type
        self._tf_checkpoint = tf_checkpoint
        self._pytorch_dump_output = pytorch_dump_output
        self._config = config
        self._finetuning_task_name = finetuning_task_name

    def run(self):
        if self._model_type == "albert":
            try:
                from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
                    convert_tf_checkpoint_to_pytorch,
                )
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "bert":
            try:
                from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
                    convert_tf_checkpoint_to_pytorch,
                )
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "funnel":
            try:
                from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
                    convert_tf_checkpoint_to_pytorch,
                )
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "t5":
            try:
                from ..models.t5.convert_t5_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "gpt":
            from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
                convert_openai_checkpoint_to_pytorch,
            )

            convert_openai_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "gpt2":
            try:
                from ..models.gpt2.convert_gpt2_original_tf_checkpoint_to_pytorch import (
                    convert_gpt2_checkpoint_to_pytorch,
                )
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_gpt2_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        elif self._model_type == "xlnet":
            try:
                from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
                    convert_xlnet_checkpoint_to_pytorch,
                )
            except ImportError:
                raise ImportError(IMPORT_ERROR_MESSAGE)

            convert_xlnet_checkpoint_to_pytorch(
                self._tf_checkpoint, self._config, self._pytorch_dump_output, self._finetuning_task_name
            )
        elif self._model_type == "xlm":
            from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
                convert_xlm_checkpoint_to_pytorch,
            )

            convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output)
        elif self._model_type == "lxmert":
            from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
                convert_lxmert_checkpoint_to_pytorch,
            )

            convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output)
        elif self._model_type == "rembert":
            from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
                convert_rembert_tf_checkpoint_to_pytorch,
            )

            convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output)
        else:


SUMMARY:
**Class Summary: ConvertCommand**

The `ConvertCommand` class is a CLI command to convert original author checkpoints to Transformers PyTorch checkpoints.

**Purpose and Functionality**
---------------------------

* The primary purpose of this class is to enable the conversion of TensorFlow checkpoints to PyTorch models for various pre-trained language models, including ALBERT, BERT, Funnel, T5, GPT, GPT2, XLM, LXMert, and REMBERT.
* The class takes in several command-line arguments, including model type, TensorFlow checkpoint path, PyTorch dump output path, configuration file path or folder, and optional fine-tuning task name.

**Key Implementation Details**
--------------------------------

* The `convert_tf_checkpoint_to_pytorch` function is called dynamically based on the model type.
* The class uses try-except blocks to handle missing import errors for specific models.

**Notable Dependencies or Requirements**
-----------------------------------------

* `base/transformers/cli/BaseTransformersCLICommand`: Base class for CLI commands
* `argparse`: Library for parsing command-line arguments
* Various PyTorch checkpoint conversion functions (e.g., `convert_albert_original_tf_checkpoint_to_pytorch`, `convert_bert_original_tf_checkpoint_to_pytorch`)

**Interfaces and Functions**
------------------------------

* `__init__`: Initializes the class with model type, TensorFlow checkpoint path, PyTorch dump output path, configuration file path or folder, and optional fine-tuning task name.
* `run`: Runs the conversion process based on the specified model type.
* `register_subcommand`: Registers the command with argparse.


key: chroma:document
string_value:
CODE:

def convert_command_factory(args: Namespace):
    """
    Factory function used to convert a model PyTorch checkpoint in a TensorFlow 2 checkpoint.

    Returns: ServeCommand
    """
    return PTtoTFCommand(
        args.model_name,
        args.local_dir,
        args.max_error,
        args.new_weights,
        args.no_pr,
        args.push,
        args.extra_commit_description,
        args.override_model_class,


SUMMARY:
**Summary**

`convert_command_factory` Factory Function:

Converts a PyTorch checkpoint to a TensorFlow 2 checkpoint.

**Key Details**

- Converts a PyTorch model checkpoint to a TensorFlow 2-friendly format using `PTtoTFCommand`.

**Method Signature**
```python
def convert_command_factory(args: Namespace)
```
- Takes `Namespace` arguments (`model_name`, `local_dir`, `max_error`, `new_weights`, `no_pr`, `push`, `extra_commit_description`, and `override_model_class`) to create a command for the conversion process.

**Notable Dependencies**

- Uses `PTtoTFCommand` class, which is not part of this snippet but assumed to be defined elsewhere in the codebase.


key: chroma:document
string_value:
CODE:

class PTtoTFCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli

        Args:
            parser: Root parser to register command-specific arguments
        """
        train_parser = parser.add_parser(
            "pt-to-tf",
            help=(
                "CLI tool to run convert a transformers model from a PyTorch checkpoint to a TensorFlow checkpoint."
                " Can also be used to validate existing weights without opening PRs, with --no-pr."
            ),
        )
        train_parser.add_argument(
            "--model-name",
            type=str,
            required=True,
            help="The model name, including owner/organization, as seen on the hub.",
        )
        train_parser.add_argument(
            "--local-dir",
            type=str,
            default="",
            help="Optional local directory of the model repository. Defaults to /tmp/{model_name}",
        )
        train_parser.add_argument(
            "--max-error",
            type=float,
            default=MAX_ERROR,
            help=(
                f"Maximum error tolerance. Defaults to {MAX_ERROR}. This flag should be avoided, use at your own risk."
            ),
        )
        train_parser.add_argument(
            "--new-weights",
            action="store_true",
            help="Optional flag to create new TensorFlow weights, even if they already exist.",
        )
        train_parser.add_argument(
            "--no-pr", action="store_true", help="Optional flag to NOT open a PR with converted weights."
        )
        train_parser.add_argument(
            "--push",
            action="store_true",
            help="Optional flag to push the weights directly to `main` (requires permissions)",
        )
        train_parser.add_argument(
            "--extra-commit-description",
            type=str,
            default="",
            help="Optional additional commit description to use when opening a PR (e.g. to tag the owner).",
        )
        train_parser.add_argument(
            "--override-model-class",
            type=str,
            default=None,
            help="If you think you know better than the auto-detector, you can specify the model class here. "
            "Can be either an AutoModel class or a specific model class like BertForSequenceClassification.",
        )
        train_parser.set_defaults(func=convert_command_factory)

    def __init__(
        self,
        model_name: str,
        local_dir: str,
        max_error: float,
        new_weights: bool,
        no_pr: bool,
        push: bool,
        extra_commit_description: str,
        override_model_class: str,
        *args,
    ):
        self._logger = logging.get_logger("transformers-cli/pt_to_tf")
        self._model_name = model_name
        self._local_dir = local_dir if local_dir else os.path.join("/tmp", model_name)
        self._max_error = max_error
        self._new_weights = new_weights
        self._no_pr = no_pr
        self._push = push
        self._extra_commit_description = extra_commit_description
        self._override_model_class = override_model_class

    def run(self):
        # TODO (joao): delete file in v4.47
        raise NotImplementedError(
            "\n\nConverting PyTorch weights to TensorFlow weights was removed in v4.43. "
            "Instead, we recommend that you convert PyTorch weights to Safetensors, an improved "
            "format that can be loaded by any framework, including TensorFlow. For more information, "
            "please see the Safetensors conversion guide: "
            "https://huggingface.co/docs/safetensors/en/convert-weights\n\n"


SUMMARY:
**Class Summary**

`PTtoTFCommand`: A command-line interface (CLI) tool in `transformers-cli` for converting PyTorch models to TensorFlow checkpoints.

**Key Functionality and Implementation Details**

* Registers the "pt-to-tf" subcommand with arguments:
	+ `model-name`: required string model name
	+ `local-dir`: optional string local directory of the model repository
	+ `max-error`: maximum error tolerance for conversion (defaults to MAX_ERROR)
	+ `new-weights`: flag to create new TensorFlow weights even if they already exist
	+ `no-pr`: flag to NOT open a PR with converted weights
	+ `push`, `extra-commit-description`, and `override-model-class` optional flags
* Initializes the command-line tool with model name, local directory, max error tolerance, flag values, and override model class (if specified)
* Provides a `run()` method that raises a NotImplementedError, indicating that converting PyTorch weights to TensorFlow weights is no longer supported in this version

**Notable Dependencies and Requirements**

* `transformers-cli`
* `base-transformers-clicommand` (inherited interface)
* External dependencies not listed

**Context**
This command-line tool is part of the `transformers-cli` package, which provides a CLI for working with Transformers models. The "pt-to-tf" subcommand allows users to convert PyTorch checkpoints to TensorFlow checkpoints, with optional flags to customize the conversion process.


key: chroma:document
string_value:
CODE:

def serve_command_factory(args: Namespace):
    """
    Factory function used to instantiate serving server from provided command line arguments.

    Returns: ServeCommand
    """
    nlp = pipeline(
        task=args.task,
        model=args.model if args.model else None,
        config=args.config,
        tokenizer=args.tokenizer,
        device=args.device,
    )


SUMMARY:
**Summary:**

 Creates a serving server instance from command line arguments.

**Main purpose and functionality:**
`serve_command_factory` is a factory function that instantiates a `ServeCommand` object based on provided command line arguments. It configures an NLP pipeline using the passed arguments and returns the `ServeCommand` instance.

**Key implementation details:**

* Configures an NLP pipeline with the following parameters:
	+ `task`
	+ `model` (optional)
	+ `config`
	+ `tokenizer`
	+ `device`

* Returns a `ServeCommand` object, which is initialized with the loaded NLP pipeline and other server configuration parameters (`host`, `port`, and `workers`).

**Notable dependencies and requirements:**

* The `pipeline` function and its related classes are not explicitly shown in this code chunk.
* External dependency: This code assumes that a namespace class (`Namespace`) is available, which is likely defined in an external module (e.g., `argparse`).


key: chroma:document
string_value:
CODE:

class ServeModelInfoResult(BaseModel):
    """
    Expose model information
    """



SUMMARY:
**Summary**

The `ServeModelInfoResult` class is a data model that exposes metadata about a resource. It inherits from the `BaseModel` and contains a single attribute `infos`, which is a dictionary.

**Primary Purpose**: Exposes metadata as key-value pairs in a structured format.

**Key Implementation Details**:

* Uses a dictionary (`dict`) to store metadata.
* Inheriting from `BaseModel` suggests that other properties or behavior may be part of this result class.

**Notable Dependencies or Requirements**:

* Inherits from `BaseModel`, implying the use of another data model or framework.
* No external dependencies listed, but may require additional libraries or frameworks depending on the used `BaseModel`.


key: chroma:document
string_value:
CODE:

class ServeTokenizeResult(BaseModel):
    """
    Tokenize result model
    """

    tokens: List[str]


SUMMARY:
**Class Summary**

`ServeTokenizeResult`
-----------

### Purpose and Functionality

The `ServeTokenizeResult` class represents the outcome of tokenization, containing lists of **tokens** (`str`) and optional **token IDs** (`int`).

### Key Implementation Details

* Uses `BaseModel` as its base class for structuring data.
* No additional specific algorithms or design choices are present in this snippet.

### Notable Dependencies or Requirements

 None.

### Interface

Inherited from: `BaseModel`

 Interface: `ServeTokenizeResult`
    Model
  - tokens [List[str]]
  - tokens_ids [Optional[List[int]]]


key: chroma:document
string_value:
CODE:

class ServeDeTokenizeResult(BaseModel):
    """
    DeTokenize result model
    """



SUMMARY:
**Class Summary: ServeDeTokenizeResult**

* **Purpose**: A data model representing the de-tokenization result of a serve operation.

**Key Implementation Details:**

* The class has one property, `text`, which is a string representing the processed text.

**Notable Dependencies/Requirements:**

* Reuses `BaseModel` from an external library or module (assuming it's named exactly as defined).
* No explicit dependencies other than `BaseModel`.
* Does not inherit any interfaces.
* Implements no specific interfaces.


key: chroma:document
string_value:
CODE:

class ServeForwardResult(BaseModel):
    """
    Forward result model
    """



SUMMARY:
**Concise Summary**

`ServeForwardResult`: A data model representing the forward result of a serving operation, containing a single attribute `output` with type `Any`, indicating the result of the request processing.


key: chroma:document
string_value:
CODE:

class ServeCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        """
        Register this command to argparse so it's available for the transformer-cli

        Args:
            parser: Root parser to register command-specific arguments
        """
        serve_parser = parser.add_parser(
            "serve", help="CLI tool to run inference requests through REST and GraphQL endpoints."
        )
        serve_parser.add_argument(
            "--task",
            type=str,
            choices=get_supported_tasks(),
            help="The task to run the pipeline on",
        )
        serve_parser.add_argument("--host", type=str, default="localhost", help="Interface the server will listen on.")
        serve_parser.add_argument("--port", type=int, default=8888, help="Port the serving will listen to.")
        serve_parser.add_argument("--workers", type=int, default=1, help="Number of http workers")
        serve_parser.add_argument("--model", type=str, help="Model's name or path to stored model.")
        serve_parser.add_argument("--config", type=str, help="Model's config name or path to stored model.")
        serve_parser.add_argument("--tokenizer", type=str, help="Tokenizer name to use.")
        serve_parser.add_argument(
            "--device",
            type=int,
            default=-1,
            help="Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)",
        )
        serve_parser.set_defaults(func=serve_command_factory)

    def __init__(self, pipeline: Pipeline, host: str, port: int, workers: int):
        self._pipeline = pipeline

        self.host = host
        self.port = port
        self.workers = workers

        if not _serve_dependencies_installed:
            raise RuntimeError(
                "Using serve command requires FastAPI and uvicorn. "
                'Please install transformers with [serving]: pip install "transformers[serving]". '
                "Or install FastAPI and uvicorn separately."
            )
        else:
            logger.info(f"Serving model over {host}:{port}")
            self._app = FastAPI(
                routes=[
                    APIRoute(
                        "/",
                        self.model_info,
                        response_model=ServeModelInfoResult,
                        response_class=JSONResponse,
                        methods=["GET"],
                    ),
                    APIRoute(
                        "/tokenize",
                        self.tokenize,
                        response_model=ServeTokenizeResult,
                        response_class=JSONResponse,
                        methods=["POST"],
                    ),
                    APIRoute(
                        "/detokenize",
                        self.detokenize,
                        response_model=ServeDeTokenizeResult,
                        response_class=JSONResponse,
                        methods=["POST"],
                    ),
                    APIRoute(
                        "/forward",
                        self.forward,
                        response_model=ServeForwardResult,
                        response_class=JSONResponse,
                        methods=["POST"],
                    ),
                ],
                timeout=600,
            )

    def run(self):
        run(self._app, host=self.host, port=self.port, workers=self.workers)

    def model_info(self):
        return ServeModelInfoResult(infos=vars(self._pipeline.model.config))

    def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)):
        """
        Tokenize the provided input and eventually returns corresponding tokens id: - **text_input**: String to
        tokenize - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer
        mapping.
        """
        try:
            tokens_txt = self._pipeline.tokenizer.tokenize(text_input)

            if return_ids:
                tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt)
                return ServeTokenizeResult(tokens=tokens_txt, tokens_ids=tokens_ids)
            else:
                return ServeTokenizeResult(tokens=tokens_txt)

        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": "", "error": str(e)})

    def detokenize(
        self,
        tokens_ids: List[int] = Body(None, embed=True),
        skip_special_tokens: bool = Body(False, embed=True),
        cleanup_tokenization_spaces: bool = Body(True, embed=True),
    ):
        """
        Detokenize the provided tokens ids to readable text: - **tokens_ids**: List of tokens ids -
        **skip_special_tokens**: Flag indicating to not try to decode special tokens - **cleanup_tokenization_spaces**:
        Flag indicating to remove all leading/trailing spaces and intermediate ones.
        """
        try:
            decoded_str = self._pipeline.tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces)
            return ServeDeTokenizeResult(model="", text=decoded_str)
        except Exception as e:
            raise HTTPException(status_code=500, detail={"model": "", "error": str(e)})

    async def forward(self, inputs=Body(None, embed=True)):
        """
        **inputs**: **attention_mask**: **tokens_type_ids**:
        """

        # Check we don't have empty string
        if len(inputs) == 0:
            return ServeForwardResult(output=[], attention=[])

        try:
            # Forward through the model
            output = self._pipeline(inputs)
            return ServeForwardResult(output=output)
        except Exception as e:


SUMMARY:
**Summary**

`ServeCommand` is a `BaseTransformersCLICommand` class responsible for serving the transformer pipeline through REST and GraphQL endpoints.

**Main Purpose and Functionality**

The class allows users to run inference requests on a pre-defined task, model, and tokenizer. It listens on specific ports and handles worker processes to serve the pipeline.

**Key Implementation Details**

* Registers subcommands with `argparse` for task selection and configuration
* Installs necessary dependencies (FastAPI and uvicorn) at runtime if not already installed
* Creates a FastAPI app with routes for model info, tokenization, detokenization, and forward requests
* Defines methods for each route:
	+ `model_info`: returns model config information
	+ `tokenize`: tokenizes input text
	+ `detokenize`: detokenizes tokens back to readable text
	+ `forward`: forwards input through the model

**Notable Dependencies and Requirements**

* FastAPI
* uvicorn
* transformers (with serving package)
* Pipelin (assuming a custom pipeline class)


key: chroma:document
string_value:
CODE:

def main():
    parser = ArgumentParser("Transformers CLI tool", usage="transformers-cli <command> [<args>]")
    commands_parser = parser.add_subparsers(help="transformers-cli command helpers")

    # Register commands
    ConvertCommand.register_subcommand(commands_parser)
    DownloadCommand.register_subcommand(commands_parser)
    EnvironmentCommand.register_subcommand(commands_parser)
    RunCommand.register_subcommand(commands_parser)
    ServeCommand.register_subcommand(commands_parser)
    UserCommands.register_subcommand(commands_parser)
    AddNewModelLikeCommand.register_subcommand(commands_parser)
    LfsCommands.register_subcommand(commands_parser)
    PTtoTFCommand.register_subcommand(commands_parser)

    # Let's go
    args = parser.parse_args()

    if not hasattr(args, "func"):
        parser.print_help()
        exit(1)

    # Run
    service = args.func(args)


SUMMARY:
**Main Function Summary**

The `main` function is the entry point of a command-line interface (CLI) tool for transforming and managing various types of data. It defines a parser with subcommand registrations for various transformers, and parses command-line arguments to determine which transformation function to execute.

**Key Implementation Details**

* Defines a parser with subcommands using `ArgumentParser` and registers multiple transformer commands.
* Parses command-line arguments and checks if the `func` attribute is present before executing the corresponding function.

**Notable Dependencies or Requirements**

* `Argparse` library for parsing command-line arguments
* Multiple transformer packages (e.g., ConvertCommand, DownloadCommand) that implement subcommands and registered with the parser.


key: chroma:document
string_value:
CODE:

class LfsCommands(BaseTransformersCLICommand):
    """
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    """

    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        enable_parser = parser.add_parser(
            "lfs-enable-largefiles",
            help=(
                "Deprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB."
            ),
        )
        enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))

        upload_parser = parser.add_parser(
            LFS_MULTIPART_UPLOAD_COMMAND,
            help=(
                "Deprecated: use `huggingface-cli` instead. "
                "Command will get called by git-lfs, do not call it directly."
            ),
        )


SUMMARY:
**LfsCommands Class Summary**

*   **Purpose**: Custom transfer agent for the `multiPart` transfer type in Git LFS, enabling large file uploads >5GB.
*   **Functionality**:
    *   Provides two CLI commands: `lfs-enable-largefiles` and `lfs-multipart-upload`.
    *   `lfs-enable-largefiles`: Configures a repository to enable large file uploads, deprecated in favor of `huggingface-cli`.
    *   `lfs-multipart-upload`: Called by Git LFS directly, not meant for direct user invocation.
*   **Key Implementation Details**:
    *   Uses the `ArgumentParser` from `Argparse` module to define CLI subcommands and arguments.
    *   Deploys two subcommands: `LfsEnableCommand` and `LfsUploadCommand`.
*   **Dependencies**: Inherited from `BaseTransformersCLICommand`, uses the `ArgumentParser` from `Argparse`. No external dependencies mentioned.

**Key Method/Function Signatures:**

*   `register_subcommand(parser)` - Registers two CLI subcommands.
*   `lfs-enable-largefiles` and `lfs-multipart-upload` functions (lambda expressions) - Invoked by the registered commands.


key: chroma:document
string_value:
CODE:

class LfsEnableCommand:
    def __init__(self, args):
        self.args = args

    def run(self):
        warnings.warn(
            "Managing repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead."
        )
        local_path = os.path.abspath(self.args.path)
        if not os.path.isdir(local_path):
            print("This does not look like a valid git repo.")
            exit(1)
        subprocess.run(
            "git config lfs.customtransfer.multipart.path transformers-cli".split(), check=True, cwd=local_path
        )
        subprocess.run(
            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
            check=True,
            cwd=local_path,
        )


SUMMARY:
**Class Summary: LfsEnableCommand**

*   **Purpose:** This class enables Local File System (LFS) support for a Git repository by configuring the `customtransfer` settings.
*

*   **Key Implementation Details:**
    *   Runs `git config` commands to set up LFS multipart upload transfer using `huggingface-cli`.
    *   Applies the uploaded `LFS_MULTIPART_UPLOAD_COMMAND` as the custom transfer argument.

       ```python
class LfsEnableCommand:
    def __init__(self, args):
        self.args = args

    def run(self):
        # Set up LFS multipart upload
        subprocess.run(
            "git config lfs.customtransfer.multipart.path transformers-cli".split(), check=True, cwd=self.args.path
        )
        subprocess.run(
            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
            check=True,
            cwd=self.args.path,
        )
```
*   **Notable Dependencies:**
    *   `subprocess` module for executing shell commands.
    *   `os` module for working with file paths and directories.
    *   ` Git commands`

       ```python
import subprocess
import os

class LfsEnableCommand:
    def __init__(self, args):
        self.args = args

# Usage of the class would involve invoking this function with a path,
# like so...

args = SomeArguments
repo_path = PathToRepository
lfs_command = LfsEnableCommand(args)
```
*   **Interfaces and Inheritance:** 
    None


key: chroma:document
string_value:
CODE:

def write_msg(msg: Dict):
    """Write out the message in Line delimited JSON."""
    msg = json.dumps(msg) + "\n"
    sys.stdout.write(msg)


SUMMARY:
**Function Summary**

`write_msg` function writes out a given message `msg` to the standard output, formatting it as line-delimited JSON.

**Key Details:**

* Formats input `msg` into a JSON string using `json.dumps()`
* Appends a newline character and writes to the standard output
* Flushes stdout buffer after writing

**Note:** This function assumes that the input `msg` is of type `Dict`.


key: chroma:document
string_value:
CODE:

def read_msg() -> Optional[Dict]:
    """Read Line delimited JSON from stdin."""
    msg = json.loads(sys.stdin.readline().strip())

    if "terminate" in (msg.get("type"), msg.get("event")):
        # terminate message received
        return None

    if msg.get("event") not in ("download", "upload"):
        logger.critical("Received unexpected message")
        sys.exit(1)



SUMMARY:
**Summary**

Function `read_msg()` reads line-delimited JSON from stdin and processes it. Its primary purpose is to validate messages received for download or upload events.

*   **Functionality**: Reads JSON input, checks message type and event, and logs critical errors if invalid.

**Key Details**

*   Reads JSON from stdin using `json.loads()` and `sys.stdin.readline()`.
*   Checks if the message indicates termination (`terminate` in type or event).
*   Validates expected events for download or upload and logs an error if unexpected.
*   Returns the validated message (`None` if terminate).

**Notable Dependencies/Requirements**

*   JSON parsing: `json`
*   Logging functions: `logger.critical()`
*   Standard input streams: `sys.stdin`
*   System exit handling: `sys.exit()`


key: chroma:document
string_value:
CODE:

class FileSlice(AbstractContextManager):
    """
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    """

    def __init__(self, filepath: str, seek_from: int, read_limit: int):
        self.filepath = filepath
        self.seek_from = seek_from
        self.read_limit = read_limit
        self.n_seen = 0

    def __enter__(self):
        self.f = open(self.filepath, "rb")
        self.f.seek(self.seek_from)
        return self

    def __len__(self):
        total_length = os.fstat(self.f.fileno()).st_size
        return min(self.read_limit, total_length - self.seek_from)

    def read(self, n=-1):
        if self.n_seen >= self.read_limit:
            return b""
        remaining_amount = self.read_limit - self.n_seen
        data = self.f.read(remaining_amount if n < 0 else min(n, remaining_amount))
        self.n_seen += len(data)
        return data

    def __iter__(self):
        yield self.read(n=4 * 1024 * 1024)

    def __exit__(self, *args):


SUMMARY:
**FileSlice Class Summary**

The `FileSlice` class is an iterable file object that reads a limited slice of a file. It provides a "file-like" interface for reading a subset of a file's contents without loading the entire file into memory.

**Key Features:**

* Reads a fixed-size slice from a file, specified by `seek_from` and `read_limit`.
* Returns an empty byte string if the read limit is reached.
* Uses `os.fstat` to estimate remaining file size.
* Iterates over the read data in chunks of 4MB.

**Notable Dependencies:**

* `os`
* `abstract`

**Implementation Details:**

* `__enter__`: Opens the file and seeks to the specified position.
* `read`: Reads up to `n` bytes from the current position, updating `self.n_seen`.
* `__iter__`: Generates chunks of 4MB using `read(n=4 * 1024 * 1024)`.

**Inherited Interfaces:**

* `AbstractContextManager` (not explicitly listed, but implied by class definition)


key: chroma:document
string_value:
CODE:

class LfsUploadCommand:
    def __init__(self, args):
        self.args = args

    def run(self):
        # Immediately after invoking a custom transfer process, git-lfs
        # sends initiation data to the process over stdin.
        # This tells the process useful information about the configuration.
        init_msg = json.loads(sys.stdin.readline().strip())
        if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
            sys.exit(1)

        # The transfer process should use the information it needs from the
        # initiation structure, and also perform any one-off setup tasks it
        # needs to do. It should then respond on stdout with a simple empty
        # confirmation structure, as follows:
        write_msg({})

        # After the initiation exchange, git-lfs will send any number of
        # transfer requests to the stdin of the transfer process, in a serial sequence.
        while True:
            msg = read_msg()
            if msg is None:
                # When all transfers have been processed, git-lfs will send
                # a terminate event to the stdin of the transfer process.
                # On receiving this message the transfer process should
                # clean up and terminate. No response is expected.
                sys.exit(0)

            oid = msg["oid"]
            filepath = msg["path"]
            completion_url = msg["action"]["href"]
            header = msg["action"]["header"]
            chunk_size = int(header.pop("chunk_size"))
            presigned_urls: List[str] = list(header.values())

            parts = []
            for i, presigned_url in enumerate(presigned_urls):
                with FileSlice(filepath, seek_from=i * chunk_size, read_limit=chunk_size) as data:
                    r = requests.put(presigned_url, data=data)
                    r.raise_for_status()
                    parts.append(
                        {
                            "etag": r.headers.get("etag"),
                            "partNumber": i + 1,
                        }
                    )
                    # In order to support progress reporting while data is uploading / downloading,
                    # the transfer process should post messages to stdout
                    write_msg(
                        {
                            "event": "progress",
                            "oid": oid,
                            "bytesSoFar": (i + 1) * chunk_size,
                            "bytesSinceLast": chunk_size,
                        }
                    )
                    # Not precise but that's ok.

            r = requests.post(
                completion_url,
                json={
                    "oid": oid,
                    "parts": parts,
                },
            )
            r.raise_for_status()



SUMMARY:
**Class Summary: LfsUploadCommand**

### Purpose and Functionality
The `LfsUploadCommand` class represents a Git LFS (Large File Storage) upload process. It handles the transfer of large files from a source to a destination, exchanging information through stdin.

### Key Implementation Details

*   Initiates the file transfer process by parsing initiation data from stdin.
*   Verifies the operation type and exchanges useful information between processes.
*   Receives transfer requests and sends corresponding responses over stdin.
*   Updates progress messages to stdout during file uploading/downloading.

### Notable Dependencies or Requirements

*   `json` for initialization message parsing
*   `requests` for making HTTP POST requests
*   `write_msg` function (not defined in the provided code) for sending progress updates and completion notifications
*   Custom `FileSlice` class for handling file slice operations
*   `sys.stdin.readline()` and `sys.exit()` functions for reading input from stdin and exiting the process

### Method Signatures and Interfaces

#### run method
```python
run(self):
    # Transfer process logic
```

This summary provides a concise overview of the key aspects of the `LfsUploadCommand` class, focusing on its purpose, implementation details, and dependencies.


key: chroma:document
string_value:
CODE:

def try_infer_format_from_ext(path: str):
    if not path:
        return "pipe"

    for ext in PipelineDataFormat.SUPPORTED_FORMATS:
        if path.endswith(ext):
            return ext

    raise Exception(
        f"Unable to determine file format from file extension {path}. "
        f"Please provide the format through --format {PipelineDataFormat.SUPPORTED_FORMATS}"


SUMMARY:
**Summary**

tries_to_infer_format_from_ext Function:

```markdown
Tries to determine the file format from its extension. 
If no specific format is specified, it infers based on supported extensions. 
Raises exception if unknown format.

Signature:
    try_infer_format_from_ext(path: str) -> str

Returns:
  Format extension (str)
```

**Key Implementation Details**

- Iterates over `PipelineDataFormat.SUPPORTED_FORMATS` to check for matching file extensions.
- Raises an exception with a hint when no match is found.

**Dependencies/Notable Requirements**

- `PipelineDataFormat`
- `Exception` class for handling unknown formats
```

Let me know if you want any further summary or clarification


key: chroma:document
string_value:
CODE:

def run_command_factory(args):
    nlp = pipeline(
        task=args.task,
        model=args.model if args.model else None,
        config=args.config,
        tokenizer=args.tokenizer,
        device=args.device,
    )
    format = try_infer_format_from_ext(args.input) if args.format == "infer" else args.format
    reader = PipelineDataFormat.from_str(
        format=format,
        output_path=args.output,
        input_path=args.input,
        column=args.column if args.column else nlp.default_input_names,
        overwrite=args.overwrite,
    )


SUMMARY:
**Summary:**

The `run_command_factory` function creates an instance of `RunCommand` by initializing two pipeline components: `nlp` (natural language processing) and `reader` (pipeline data format). The function takes input arguments to configure both pipelines, including model selection, output paths, and input formatting. It returns a fully configured `RunCommand` object.

**Key Implementation Details:**

* NLP pipeline initialization with customizable tasks, models, and configuration.
* Format inference from file extension for the reader pipeline.
* Usage of PipelineDataFormat.from_str to create reader components.

**Notable Dependencies/Requirements:**

* No external dependencies are explicitly referenced in this snippet. However, it assumes presence of `pipeline` and `RunCommand` functions/classes, likely from the NLTK library.


key: chroma:document
string_value:
CODE:

class RunCommand(BaseTransformersCLICommand):
    def __init__(self, nlp: Pipeline, reader: PipelineDataFormat):
        self._nlp = nlp
        self._reader = reader

    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        run_parser = parser.add_parser("run", help="Run a pipeline through the CLI")
        run_parser.add_argument("--task", choices=get_supported_tasks(), help="Task to run")
        run_parser.add_argument("--input", type=str, help="Path to the file to use for inference")
        run_parser.add_argument("--output", type=str, help="Path to the file that will be used post to write results.")
        run_parser.add_argument("--model", type=str, help="Name or path to the model to instantiate.")
        run_parser.add_argument("--config", type=str, help="Name or path to the model's config to instantiate.")
        run_parser.add_argument(
            "--tokenizer", type=str, help="Name of the tokenizer to use. (default: same as the model name)"
        )
        run_parser.add_argument(
            "--column",
            type=str,
            help="Name of the column to use as input. (For multi columns input as QA use column1,columns2)",
        )
        run_parser.add_argument(
            "--format",
            type=str,
            default="infer",
            choices=PipelineDataFormat.SUPPORTED_FORMATS,
            help="Input format to read from",
        )
        run_parser.add_argument(
            "--device",
            type=int,
            default=-1,
            help="Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)",
        )
        run_parser.add_argument("--overwrite", action="store_true", help="Allow overwriting the output file.")
        run_parser.set_defaults(func=run_command_factory)

    def run(self):
        nlp, outputs = self._nlp, []

        for entry in self._reader:
            output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry)
            if isinstance(output, dict):
                outputs.append(output)
            else:
                outputs += output

        # Saving data
        if self._nlp.binary_output:
            binary_path = self._reader.save_binary(outputs)
            logger.warning(f"Current pipeline requires output to be in binary format, saving at {binary_path}")
        else:


SUMMARY:
**Summary**

`RunCommand` is a CLI command for running a natural language processing (NLP) pipeline. Its primary purpose is to execute the pipeline through the provided `PipelineDataFormat` and save the results.

**Key Implementation Details**

* Registers subcommand `run` with several optional arguments:
	+ `task`, `input`, `output`, `model`, `config`, `tokenizer`, `column`, `format`, `device`
		- Corresponding default values for some arguments
* Uses a predefined factory to create the actual execution process (`run_command_factory`)
* Executes the pipeline by iterating over the input data and saving the results
* Supports binary output format when specified

**Notable Dependencies or Requirements**

* `Pipeline` class from `BaseTransformersCLICommand`
* `PipelineDataFormat` interface or class
* `TokenizationFactory` (assuming a tokenization setup for tokenizer usage)
* External dependency on logger and binary saving functionality

**Interfaces/Classes Used:**

* `Pipeline`
* `PipelineDataFormat`
* `ArgumentParser`
* `BaseTransformersCLICommand`


key: chroma:document
string_value:
CODE:

def info_command_factory(_):


SUMMARY:
**Summary**

Creates and returns an instance of `EnvironmentCommand`, a command that likely interacts with an environment-specific configuration.

**Key Details**

* Function name: `info_command_factory`
* Return type: `EnvironmentCommand` instance
* Primary purpose: Returns a command instance for information-related operations

Note: The actual implementation of the returned `EnvironmentCommand` is not shown in this code chunk.


key: chroma:document
string_value:
CODE:

def download_command_factory(args):


SUMMARY:
**Summary**

`download_command_factory` Function:

* Creates an instance of `EnvironmentCommand` with the provided `accelerate_config_file`.
* Main purpose: Factory function for generating EnvironmentCommand instances based on acceleration configuration.

**Implementation Details**

* Uses the `args` object to extract the `accelerate_config_file` parameter.
* Directly returns an instance of `EnvironmentCommand`, bypassing a separate constructor call.

**Notable Dependencies/Requirements**

* None explicitly mentioned; `EnvironmentCommand` is assumed to be defined elsewhere in the codebase.


key: chroma:document
string_value:
CODE:

class EnvironmentCommand(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        download_parser = parser.add_parser("env")
        download_parser.set_defaults(func=info_command_factory)
        download_parser.add_argument(
            "--accelerate-config_file",
            default=None,
            help="The accelerate config file to use for the default values in the launching script.",
        )
        download_parser.set_defaults(func=download_command_factory)

    def __init__(self, accelerate_config_file, *args) -> None:
        self._accelerate_config_file = accelerate_config_file

    def run(self):
        safetensors_version = "not installed"
        if is_safetensors_available():
            import safetensors

            safetensors_version = safetensors.__version__
        elif importlib.util.find_spec("safetensors") is not None:
            import safetensors

            safetensors_version = f"{safetensors.__version__} but is ignored because of PyTorch version too old."

        accelerate_version = "not installed"
        accelerate_config = accelerate_config_str = "not found"
        if is_accelerate_available():
            import accelerate
            from accelerate.commands.config import default_config_file, load_config_from_file

            accelerate_version = accelerate.__version__
            # Get the default from the config file.
            if self._accelerate_config_file is not None or os.path.isfile(default_config_file):
                accelerate_config = load_config_from_file(self._accelerate_config_file).to_dict()

            accelerate_config_str = (
                "\n".join([f"\t- {prop}: {val}" for prop, val in accelerate_config.items()])
                if isinstance(accelerate_config, dict)
                else f"\t{accelerate_config}"
            )

        pt_version = "not installed"
        pt_cuda_available = "NA"
        if is_torch_available():
            import torch

            pt_version = torch.__version__
            pt_cuda_available = torch.cuda.is_available()
            pt_npu_available = is_torch_npu_available()

        tf_version = "not installed"
        tf_cuda_available = "NA"
        if is_tf_available():
            import tensorflow as tf

            tf_version = tf.__version__
            try:
                # deprecated in v2.1
                tf_cuda_available = tf.test.is_gpu_available()
            except AttributeError:
                # returns list of devices, convert to bool
                tf_cuda_available = bool(tf.config.list_physical_devices("GPU"))

        flax_version = "not installed"
        jax_version = "not installed"
        jaxlib_version = "not installed"
        jax_backend = "NA"
        if is_flax_available():
            import flax
            import jax
            import jaxlib

            flax_version = flax.__version__
            jax_version = jax.__version__
            jaxlib_version = jaxlib.__version__
            jax_backend = jax.lib.xla_bridge.get_backend().platform

        info = {
            "`transformers` version": version,
            "Platform": platform.platform(),
            "Python version": platform.python_version(),
            "Huggingface_hub version": huggingface_hub.__version__,
            "Safetensors version": f"{safetensors_version}",
            "Accelerate version": f"{accelerate_version}",
            "Accelerate config": f"{accelerate_config_str}",
            "PyTorch version (GPU?)": f"{pt_version} ({pt_cuda_available})",
            "Tensorflow version (GPU?)": f"{tf_version} ({tf_cuda_available})",
            "Flax version (CPU?/GPU?/TPU?)": f"{flax_version} ({jax_backend})",
            "Jax version": f"{jax_version}",
            "JaxLib version": f"{jaxlib_version}",
            "Using distributed or parallel set-up in script?": "<fill in>",
        }
        if is_torch_available():
            if pt_cuda_available:
                info["Using GPU in script?"] = "<fill in>"
                info["GPU type"] = torch.cuda.get_device_name()
            elif pt_npu_available:
                info["Using NPU in script?"] = "<fill in>"
                info["NPU type"] = torch.npu.get_device_name()
                info["CANN version"] = torch.version.cann

        print("\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n")
        print(self.format_dict(info))

        return info

    @staticmethod
    def format_dict(d):


SUMMARY:
**EnvironmentCommand Class Summary**

### Purpose and Functionality

The `EnvironmentCommand` class is a CLI command that provides information about the environment and its installed dependencies. It uses various external libraries (e.g., `safetensors`, `accelerate`, `torch`, `tensorflow`, `flax`, `jax`) to gather version numbers, platform information, and GPU availability.

### Implementation Details

* The `register_subcommand` method adds a parser for the "env" command, which runs when the "--env" flag is used.
* The `run` method checks for available libraries, collects their versions and configurations, and formats the results as a string.

### Key Method Signatures and Interfaces

* `is_safetensors_available()`
* `importlib.util.find_spec("safetensors")`
* `download_command_factory`
* `info_command_factory`
* `load_config_from_file`

### Notable Dependencies or Requirements

* External libraries: `safetensors`, `accelerate`, `torch`, `tensorflow`, `flax`, `jax`
* Inherited interfaces:
	+ `ArgumentParser` (from ` Click`)
	+ `BaseTransformersCLICommand`


key: chroma:document
string_value:
CODE:

class UserCommands(BaseTransformersCLICommand):
    @staticmethod
    def register_subcommand(parser: ArgumentParser):
        login_parser = parser.add_parser("login", help="Log in using the same credentials as on huggingface.co")
        login_parser.set_defaults(func=lambda args: LoginCommand(args))
        whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.")
        whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args))
        logout_parser = parser.add_parser("logout", help="Log out")
        logout_parser.set_defaults(func=lambda args: LogoutCommand(args))

        # new system: git-based repo system
        repo_parser = parser.add_parser(
            "repo",
            help="Deprecated: use `huggingface-cli` instead. Commands to interact with your huggingface.co repos.",
        )
        repo_subparsers = repo_parser.add_subparsers(
            help="Deprecated: use `huggingface-cli` instead. huggingface.co repos related commands"
        )
        repo_create_parser = repo_subparsers.add_parser(
            "create", help="Deprecated: use `huggingface-cli` instead. Create a new repo on huggingface.co"
        )
        repo_create_parser.add_argument(
            "name",
            type=str,
            help="Name for your model's repo. Will be namespaced under your username to build the model id.",
        )
        repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.")
        repo_create_parser.add_argument("-y", "--yes", action="store_true", help="Optional: answer Yes to the prompt")


SUMMARY:
**Summary:**

`UserCommands` class enables subcommands for user interaction, including logging in/out and managing Hugging Face repositories. It defines:

*   Two basic commands (`login` and `whoami`) that delegate to specific `LoginCommand` and `WhoamiCommand` respectively.
*   A deprecated legacy repository-related command group (`repo`) with a create parser.

**Key Details:**

*   Subcommands:
    *   `login`: Logs in using the same credentials as on Hugging Face Co (`func=lambda args: LoginCommand(args)`)
    *   `whoami`: Finds out which Hugging Face Co account is logged in as (`func=lambda args: WhoamiCommand(args)`)
    *   `logout`: Logs out (`func=lambda args: LogoutCommand(args)`)

*   Legacy Repository Commands:
    *   `repo create` (deprecated, use Hugging Face CLI instead): 
        +   Creates a new repository on Hugging Face Co (`repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))`)
        +   Parameters:
            -   `name`: The name of the model's repo
            -   `-y` or `--yes`: Optional, answer Yes to the prompt

*   External Dependencies:

    +   `BaseTransformersCLICommand`
    +   `ArgumentParser`
    +   `LoginCommand`
    +   `WhoamiCommand`
    +   `LogoutCommand`
    +   `RepoCreateCommand`


key: chroma:document
string_value:
CODE:

class ANSI:
    """
    Helper for en.wikipedia.org/wiki/ANSI_escape_code
    """

    _bold = "\u001b[1m"
    _red = "\u001b[31m"
    _gray = "\u001b[90m"
    _reset = "\u001b[0m"

    @classmethod
    def bold(cls, s):
        return f"{cls._bold}{s}{cls._reset}"

    @classmethod
    def red(cls, s):
        return f"{cls._bold}{cls._red}{s}{cls._reset}"

    @classmethod
    def gray(cls, s):


SUMMARY:
**Summary: ANSI Color Code Helper Class**

The `ANSI` class provides a simple way to apply colored text in terminal outputs. It uses ANSI escape codes to color text and includes methods for bold, red, and gray text.

**Key Implementation Details:**

* `bold(s)`, `red(s)`, and `gray(s)` methods use ANSI escape codes (`\u001b[1m`, `\u001b[31m`, and `\u001b[90m`) to format the input string with the corresponding color.
* `_reset` code is used to reset the text color to default.

**Notable Dependencies or Requirements:**

* None. This class only relies on the terminal's support for ANSI escape codes.

```
class ANSI:
    @classmethod
    def bold(cls, s):
        return f"{cls._bold}{s}{cls._reset}"

    @classmethod
    def red(cls, s):
        return f"{cls._bold}{cls._red}{s}{cls._reset}"

    @classmethod
    def gray(cls, s):
        return f"{cls._gray}{s}{cls._reset}"
```


key: chroma:document
string_value:
CODE:

def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
    """
    Inspired by:

    - stackoverflow.com/a/8356620/593036
    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
    """
    col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
    row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
    lines = []
    lines.append(row_format.format(*headers))
    lines.append(row_format.format(*["-" * w for w in col_widths]))
    for row in rows:
        lines.append(row_format.format(*row))


SUMMARY:
**Function Summary: `tabulate`**

* **Purpose:** Formats a table from tabular data into a string representation.

* **Key Implementation Details:**
	+ Calculates column widths using the maximum length of values in each column.
	+ Uses Python's string formatting to create a tabular layout with specified columns.
* **Notable Dependencies/Requirements:**
	+ `List` and `Union` types from the standard library.
	+ No external dependencies, but it references stack overflow links for inspiration.

**Key Method Signature:**

`tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str`

This function takes in a list of rows (where each row is a list of strings or integers) and a list of column headers. It returns a string representation of the tabulated data.

**Design Choices:**

* Uses Python's built-in string formatting to create a tabular layout, which provides flexibility and ease of use.
* Calculates column widths using `zip` and list comprehension, which is concise and efficient.
* Appends multiple lines to the result string, which demonstrates a clear understanding of how to build up a complex data structure.


key: chroma:document
string_value:
CODE:

class BaseUserCommand:
    def __init__(self, args):


SUMMARY:
**Summary:**

The `BaseUserCommand` class serves as the foundation for user commands. It initializes with an `args` parameter, storing it as an instance variable, allowing for data transfer between command instances.

**Key Details:**

* **Implementation:** Initializer function (`__init__`) sets up the base structure.
* **Primary Purpose:** Facilitates inheritance and polymorphism for user commands.
* **Important Interface:** None.

**Notable Dependencies/Requirements:**

* `args` parameter (instance variable)
* No external dependencies explicitly stated.


key: chroma:document
string_value:
CODE:

class LoginCommand(BaseUserCommand):
    def run(self):
        print(
            ANSI.red(
                "ERROR! `huggingface-cli login` uses an outdated login mechanism "
                "that is not compatible with the Hugging Face Hub backend anymore. "
                "Please use `huggingface-cli login instead."
            )


SUMMARY:
**Summary:**

`LoginCommand` class informs the user about the incompatibility of using an outdated login mechanism with the current Hugging Face Hub backend, and recommends updating to `huggingface-cli login`.

**Key Details:**

* Class: `LoginCommand`
* Uses: outdated login mechanism (incompatible with Hugging Face Hub backend)
* Recommendation: use `huggingface-cli login` instead
* Output:
	+ Error message printed in red ANSI color

**Interfaces and Dependencies:**

* `BaseUserCommand`: inherited from
* None noted


key: chroma:document
string_value:
CODE:

class WhoamiCommand(BaseUserCommand):
    def run(self):
        print(
            ANSI.red(
                "WARNING! `transformers-cli whoami` is deprecated and will be removed in v5. Please use "
                "`huggingface-cli whoami` instead."
            )
        )
        token = HfFolder.get_token()
        if token is None:
            print("Not logged in")
            exit()
        try:
            user, orgs = whoami(token)
            print(user)
            if orgs:
                print(ANSI.bold("orgs: "), ",".join(orgs))
        except HTTPError as e:
            print(e)
            print(ANSI.red(e.response.text))


SUMMARY:
**Class Summary**

`WhoamiCommand` is a command class that retrieves information about the current user on the Hugging Face platform. Its primary purpose is to display the user's details, including their ID and organizations they are part of (if applicable), while logged in using the `HfFolder.get_token()` method.

**Key Implementation Details**

* Displays deprecation warning for using deprecated `transformers-cli whoami` command.
* Retrieves user information using the `whoami` function and logs them to the console.
* Handles exceptions, such as HTTP errors, by printing error messages and exiting with a non-zero status code.

**Notable Dependencies/Requirements**

* `HfFolder`: referenced for getting the logged-in token.
* `transformers-cli` (deprecated): referenced in deprecation warning.
* `HTTPError`: exception type handled by capturing it from `e`.


key: chroma:document
string_value:
CODE:

class LogoutCommand(BaseUserCommand):
    def run(self):
        print(
            ANSI.red(
                "ERROR! `transformers-cli logout` uses an outdated logout mechanism "
                "that is not compatible with the Hugging Face Hub backend anymore. "
                "Please use `huggingface-cli logout instead."
            )


SUMMARY:
**Summary:**
```python
LogoutCommand(BaseUserCommand):
    Runs a command to log out, printing an error message due to outdated logout mechanism compatibility issues with the Hugging Face Hub backend. 
```

**Implementation Details:**

* Calls `print()` with an ANSI-red colored error message.

**Notable Dependencies:**

* `BaseUserCommand` class
* `ANSI` library for colorized output

Note: This summary captures the essence of the code, focusing on its purpose and key implementation details.


key: chroma:document
string_value:
CODE:

class RepoCreateCommand(BaseUserCommand):
    def run(self):
        print(
            ANSI.red(
                "WARNING! Managing repositories through transformers-cli is deprecated. "
                "Please use `huggingface-cli` instead."
            )
        )
        token = HfFolder.get_token()
        if token is None:
            print("Not logged in")
            exit(1)
        try:
            stdout = subprocess.check_output(["git", "--version"]).decode("utf-8")
            print(ANSI.gray(stdout.strip()))
        except FileNotFoundError:
            print("Looks like you do not have git installed, please install.")

        try:
            stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
            print(ANSI.gray(stdout.strip()))
        except FileNotFoundError:
            print(
                ANSI.red(
                    "Looks like you do not have git-lfs installed, please install."
                    " You can install from https://git-lfs.github.com/."
                    " Then run `git lfs install` (you only have to do this once)."
                )
            )
        print("")

        user, _ = whoami(token)
        namespace = self.args.organization if self.args.organization is not None else user
        full_name = f"{namespace}/{self.args.name}"
        print(f"You are about to create {ANSI.bold(full_name)}")

        if not self.args.yes:
            choice = input("Proceed? [Y/n] ").lower()
            if not (choice == "" or choice == "y" or choice == "yes"):
                print("Abort")
                exit()
        try:
            url = create_repo(repo_id=full_name, token=token)
        except HTTPError as e:
            print(e)
            print(ANSI.red(e.response.text))
            exit(1)
        print("\nYour repo now lives at:")
        print(f"  {ANSI.bold(url)}")
        print("\nYou can clone it locally with the command below, and commit/push as usual.")
        print(f"\n  git clone {url}")


SUMMARY:
**RepoCreateCommand Class Summary**

**Purpose and Functionality**
The `RepoCreateCommand` class is used to create a new repository using the Git Large File Storage (Git LFS) version control system. It authenticates the user, checks for installation of required tools, and creates a new repository.

**Key Implementation Details**

* Uses `HfFolder.get_token()` to authenticate the user
* Displays version information for `git` and `git-lfs`
* Calculates the repository full name based on namespace and repository name
* Prompts the user to confirm creation before proceeding
* Calls the `create_repo` function to create the new repository

**Notable Dependencies and Requirements**

* `git` and `git-lfs` are required tools
* `HfFolder` class is used for authentication
* `whoami` function is used to authenticate the user
* `create_repo` function is called to create a new repository, which may throw an `HTTPError`

**Interface/Method Signatures**

* No public methods; all implementation details are within the `__init__` method (not shown in this code snippet).


key: chroma:document
string_value:
CODE:

def list_field(default=None, metadata=None):


SUMMARY:
**Function Summary**

`list_field` creates and returns a Django `ModelField`, providing a default value through a lambda function. It also accepts optional metadata parameters.

**Key Implementation Details**

* Uses Django's `default_factory` parameter to set the default value of the field
* Applies metadata to the created field using the `metadata` parameter

**Notable Dependencies**

* Django (specifically, its ORM and Field classes)


key: chroma:document
string_value:
CODE:

def run_with_tf_optimizations(do_eager_mode: bool, use_xla: bool):
    def run_func(func):
        @wraps(func)
        def run_in_eager_mode(*args, **kwargs):
            return func(*args, **kwargs)

        @wraps(func)
        @tf.function(experimental_compile=use_xla)
        def run_in_graph_mode(*args, **kwargs):
            return func(*args, **kwargs)

        if do_eager_mode is True:
            if use_xla is not False:
                raise ValueError(
                    "Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."
                )
            return run_in_eager_mode
        else:
            return run_in_graph_mode



SUMMARY:
**Summary:**

The `run_with_tf_optimizations` function enables or disables TensorFlow optimizations (XLA) in a given model execution. It takes two boolean inputs: `do_eager_mode` and `use_xla`.

**Key Functionality:**

- Runs the provided function (`func`) either in Eager mode or Graph mode based on the input value of `do_eager_mode`.
- When `do_eager_mode` is True, it raises an error if `use_xla` is set to True.
- Uses TensorFlow's `tf.function` decorator with `experimental_compile=use_xla` to enable XLA optimizations in Graph mode.

**Implementation Details:**

- The function utilizes a wrapper (`run_func`) that takes the input function (`func`) and returns another wrapper (`run_in_eager_mode` or `run_in_graph_mode`).
- The `@wraps(func)` decorator is used to preserve the original function's name and docstring.
- The `tf.function` decorator with the `experimental_compile=use_xla` argument enables XLA optimizations in Graph mode.

**Dependencies and Requirements:**

- TensorFlow (`tf`) with support for XLA (Experimental).


key: chroma:document
string_value:
CODE:

def random_input_ids(batch_size: int, sequence_length: int, vocab_size: int) -> ["tf.Tensor"]:
    rng = random.Random()
    values = [rng.randint(0, vocab_size - 1) for i in range(batch_size * sequence_length)]


SUMMARY:
**Function Summary:**

`random_input_ids`: generates a batch of randomly sampled integer indices from a vocabulary size, shaped as input tensor for transformer model.

**Key Implementation Details:**

* Utilizes `tf.constant` to convert list of integers into a TensorFlow tensor with shape `(batch_size, sequence_length)` and dtype `tf.int32`.
* Random integers are generated using `random.Random()` to ensure reproducibility.
* Input size defaults to batch size x sequence length.

**Notable Dependencies/Requirements:**

* `tensorflow (tf)`
* `random` module


key: chroma:document
string_value:
CODE:

class TensorFlowBenchmark(Benchmark):
    args: TensorFlowBenchmarkArguments
    configs: PretrainedConfig
    framework: str = "TensorFlow"

    @property
    def framework_version(self):
        return tf.__version__

    def _inference_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
        # initialize GPU on separate process
        strategy = self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow.")
        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
        return self._measure_speed(_inference)

    def _train_speed(self, model_name: str, batch_size: int, sequence_length: int) -> float:
        strategy = self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow.")
        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
        return self._measure_speed(_train)

    def _inference_memory(
        self, model_name: str, batch_size: int, sequence_length: int
    ) -> [Memory, Optional[MemorySummary]]:
        # initialize GPU on separate process
        if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx], True)
        strategy = self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow.")
        _inference = self._prepare_inference_func(model_name, batch_size, sequence_length)
        return self._measure_memory(_inference)

    def _train_memory(
        self, model_name: str, batch_size: int, sequence_length: int
    ) -> [Memory, Optional[MemorySummary]]:
        if self.args.is_gpu:
            tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx], True)
        strategy = self.args.strategy
        if strategy is None:
            raise ValueError("A device strategy has to be initialized before using TensorFlow.")

        _train = self._prepare_train_func(model_name, batch_size, sequence_length)
        return self._measure_memory(_train)

    def _prepare_inference_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
        config = self.config_dict[model_name]

        if self.args.fp16:
            raise NotImplementedError("Mixed precision is currently not supported.")

        has_model_class_in_config = (
            hasattr(config, "architectures")
            and isinstance(config.architectures, list)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
                model_class = "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                transformers_module = __import__("transformers", fromlist=[model_class])
                model_cls = getattr(transformers_module, model_class)
                model = model_cls(config)
            except ImportError:
                raise ImportError(
                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
                    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."
                )
        else:
            model = TF_MODEL_MAPPING[config.__class__](config)

        # encoder-decoder has vocab size saved differently
        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
        input_ids = random_input_ids(batch_size, sequence_length, vocab_size)

        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
        def encoder_decoder_forward():
            return model(input_ids, decoder_input_ids=input_ids, training=False)

        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
        def encoder_forward():
            return model(input_ids, training=False)

        _inference = encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

        return _inference

    def _prepare_train_func(self, model_name: str, batch_size: int, sequence_length: int) -> Callable[[], None]:
        config = self.config_dict[model_name]

        if self.args.eager_mode is not False:
            raise ValueError("Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.")

        if self.args.fp16:
            raise NotImplementedError("Mixed precision is currently not supported.")

        has_model_class_in_config = (
            hasattr(config, "architectures")
            and isinstance(config.architectures, list)
            and len(config.architectures) > 0
        )
        if not self.args.only_pretrain_model and has_model_class_in_config:
            try:
                model_class = "TF" + config.architectures[0]  # prepend 'TF' for tensorflow model
                transformers_module = __import__("transformers", fromlist=[model_class])
                model_cls = getattr(transformers_module, model_class)
                model = model_cls(config)
            except ImportError:
                raise ImportError(
                    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
                    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."
                )
        else:
            model = TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](config)

        # encoder-decoder has vocab size saved differently
        vocab_size = config.vocab_size if hasattr(config, "vocab_size") else config.encoder.vocab_size
        input_ids = random_input_ids(batch_size, sequence_length, vocab_size)

        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
        def encoder_decoder_train():
            loss = model(input_ids, decoder_input_ids=input_ids, labels=input_ids, training=True)[0]
            gradients = tf.gradients(loss, model.trainable_variables)
            return gradients

        @run_with_tf_optimizations(self.args.eager_mode, self.args.use_xla)
        def encoder_train():
            loss = model(input_ids, labels=input_ids, training=True)[0]
            gradients = tf.gradients(loss, model.trainable_variables)
            return gradients

        _train = encoder_decoder_train if config.is_encoder_decoder else encoder_train

        return _train

    def _measure_speed(self, func) -> float:
        with self.args.strategy.scope():
            try:
                if self.args.is_tpu or self.args.use_xla:
                    # run additional 10 times to stabilize compilation for tpu
                    logger.info("Do inference on TPU. Running model 5 times to stabilize compilation")
                    timeit.repeat(func, repeat=1, number=5)

                # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                runtimes = timeit.repeat(
                    func,
                    repeat=self.args.repeat,
                    number=10,
                )

                return min(runtimes) / 10.0
            except ResourceExhaustedError as e:
                self.print_fn(f"Doesn't fit on GPU. {e}")

    def _measure_memory(self, func: Callable[[], None]) -> [Memory, MemorySummary]:
        logger.info(
            "Note that TensorFlow allocates more memory than "
            "it might need to speed up computation. "
            "The memory reported here corresponds to the memory "
            "reported by `nvidia-smi`, which can vary depending "
            "on total available memory on the GPU that is used."
        )
        with self.args.strategy.scope():
            try:
                if self.args.trace_memory_line_by_line:
                    if not self.args.eager_mode:
                        raise ValueError(
                            "`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"
                            " consumption line by line."
                        )
                    trace = start_memory_tracing("transformers")

                if self.args.is_tpu:
                    # tpu
                    raise NotImplementedError(
                        "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"
                        " with `args.memory=False`"
                    )
                elif self.args.is_gpu:
                    # gpu
                    if not is_py3nvml_available():
                        logger.warning(
                            "py3nvml not installed, we won't log GPU memory usage. "
                            "Install py3nvml (pip install py3nvml) to log information about GPU."
                        )
                        memory = "N/A"
                    else:
                        logger.info(
                            "Measuring total GPU usage on GPU device. Make sure to not have additional processes"
                            " running on the same GPU."
                        )
                        # init nvml
                        nvml.nvmlInit()
                        func()
                        handle = nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx)
                        meminfo = nvml.nvmlDeviceGetMemoryInfo(handle)
                        max_bytes_in_use = meminfo.used
                        memory = Memory(max_bytes_in_use)
                        # shutdown nvml
                        nvml.nvmlShutdown()
                else:
                    # cpu
                    if self.args.trace_memory_line_by_line:
                        logger.info(
                            "When enabling line by line tracing, the max peak memory for CPU is inaccurate in"
                            " TensorFlow."
                        )
                        memory = None
                    else:
                        memory_bytes = measure_peak_memory_cpu(func)
                        memory = Memory(memory_bytes) if isinstance(memory_bytes, int) else memory_bytes
                if self.args.trace_memory_line_by_line:
                    summary = stop_memory_tracing(trace)
                    if memory is None:
                        memory = summary.total
                else:
                    summary = None

                return memory, summary
            except ResourceExhaustedError as e:
                self.print_fn(f"Doesn't fit on GPU. {e}")


SUMMARY:
This code appears to be part of a larger class or module for benchmarking and measuring the performance of transformer models, likely in a deep learning context. The specific methods of the class are:

1. **Model-related methods**: These include various functions related to training, inference, and benchmarking of transformer models.
2. **Benchmarking methods**:
	* `_measure_speed`: Measures the execution time of a given function.
	* `_measure_memory`: Measures the memory usage of a given function.

**Key takeaways:**

1. The code uses various libraries such as `timeit`, `nvidia-smi`, and `py3nvml` to measure performance and memory usage.
2. It supports both GPU and CPU benchmarking, with some specific configurations only applicable to certain devices or environments (e.g., TPU, multiple GPUs).
3. The code includes various logging statements to provide feedback on the progress of the benchmarks.

**Suggestions:**

1. **Code organization**: Some methods seem duplicated in terms of functionality or scope. Consider refactoring these parts for better modularity and reusability.
2. **Type hints and documentation**: Adding explicit type hints and comments (preferably with Dox) to describe each method's purpose, input parameters, output values, and any dependencies would make the code more readable and maintainable.
3. **Exception handling**: While some exceptions are caught and handled, it might be beneficial to include additional logging statements for non-catchable or unhandled exceptions to provide a clearer error message when applicable.

**Additional information:**

If possible, include any specific requirements, constraints, or optimizations that the code should adhere to (e.g., PyTorch, TensorFlow, performance metrics). This would enable an individual to better understand the context and implications of the provided code.


key: chroma:document
string_value:
CODE:

def separate_process_wrapper_fn(func: Callable[[], None], do_multi_processing: bool) -> Callable[[], None]:
    """
    This function wraps another function into its own separated process. In order to ensure accurate memory
    measurements it is important that the function is executed in a separate process

    Args:
        - `func`: (`callable`): function() -> ... generic function which will be executed in its own separate process
        - `do_multi_processing`: (`bool`) Whether to run function on separate process or not
    """

    def multi_process_func(*args, **kwargs):
        # run function in an individual
        # process to get correct memory
        def wrapper_func(queue: Queue, *args):
            try:
                result = func(*args)
            except Exception as e:
                logger.error(e)
                print(e)
                result = "N/A"
            queue.put(result)

        queue = Queue()
        p = Process(target=wrapper_func, args=[queue] + list(args))
        p.start()
        result = queue.get()
        p.join()
        return result

    if do_multi_processing:
        logger.info(f"Function {func} is executed in its own process...")
        return multi_process_func
    else:


SUMMARY:
**Summary: Separate Function Execution**

The `separate_process_wrapper_fn` function wraps another function (`func`) to execute it in a separate process. It ensures accurate memory measurements by running the function individually.

**Key Details:**

* Wraps `func` in a separate process if `do_multi_processing` is True
* Executes `func` with individual arguments in a separate process using multiprocessing

**Notable Function Signatures:**

* `wrapper_func(Queue, *args)`: wrapper function for executing `func` in a separate process
* `multi_process_func(*args, **kwargs)`: main function that runs the wrapped function in a separate process

**External Dependencies:**

* `Queue` and `Process` from multiprocessing library for parallel processing
* `logger` module for logging errors


key: chroma:document
string_value:
CODE:

def is_memory_tracing_enabled():
    global _is_memory_tracing_enabled


SUMMARY:
**Code Summary:**

`is_memory_tracing_enabled()` Function

*   **Purpose**: Returns whether memory tracing is currently enabled in the application.

*   **Key Implementation Details**:
    *   Uses a global variable `_is_memory_tracing_enabled` to store the enablement status.
    *   Simply returns the value of this global variable without any logic or calculation.

*   **Notable Dependencies or Requirements**:
    +   Global variable `_is_memory_tracing_enabled`

This function has no complex logic and solely relies on a global variable for its return value.


key: chroma:document
string_value:
CODE:

class Frame(NamedTuple):
    """
    `Frame` is a NamedTuple used to gather the current frame state. `Frame` has the following fields:

        - 'filename' (string): Name of the file currently executed
        - 'module' (string): Name of the module currently executed
        - 'line_number' (int): Number of the line currently executed
        - 'event' (string): Event that triggered the tracing (default will be "line")
        - 'line_text' (string): Text of the line in the python script
    """

    filename: str
    module: str
    line_number: int
    event: str


SUMMARY:
**Class Summary**

`Frame`: A NamedTuple representing the current frame state, comprising file and module information, execution metadata, and Python script text. The primary purpose is to gather and store frame data for tracing purposes.

**Key Implementation Details**

- `event`: An optional string field defaulting to "line" to track event triggers.
- Four specific fields: `filename`, `module`, `line_number`, `line_text` (strings), providing essential frame state information.

**Notable Dependencies or Requirements**

- Referenced as a NamedTuple from the `dataclasses` module using `namedtuple()`.
- No explicit external dependencies.


key: chroma:document
string_value:
CODE:

class UsedMemoryState(NamedTuple):
    """
    `UsedMemoryState` are named tuples with the following fields:

        - 'frame': a `Frame` namedtuple (see below) storing information on the current tracing frame (current file,
          location in current file)
        - 'cpu_memory': CPU RSS memory state *before* executing the line
        - 'gpu_memory': GPU used memory *before* executing the line (sum for all GPUs or for only `gpus_to_trace` if
          provided)
    """

    frame: Frame
    cpu_memory: int


SUMMARY:
**UsedMemoryState Class Summary**

*   **Purpose and Functionality**: The `UsedMemoryState` class represents the memory state before executing a line of code, consisting of:
    *   `frame`: current tracing frame (file and location)
    *   `cpu_memory`: CPU RSS memory usage
    *   `gpu_memory`: GPU used memory (combined or per-GPU, depending on `gpus_to_trace`)

*   **Key Implementation Details**:
    *   The class utilizes a named tuple to store memory state information.

*   **Notable Dependencies or Requirements**:
    *   `Frame` namedtuple is referenced.
    *   `gpus_to_trace` (an optional parameter) influences GPU memory calculation.
    
**Example Code**
```python
from collections import namedtuple

class UsedMemoryState(NamedTuple):
    pass  # implementation details omitted for brevity
```
Note: The actual implementation of the `Frame` namedtuple and `gpus_to_trace` variable are not provided in the given code chunk.


key: chroma:document
string_value:
CODE:

class Memory(NamedTuple):
    """
    `Memory` NamedTuple have a single field `bytes` and you can get a human readable str of the number of mega bytes by
    calling `__repr__`

        - `byte` (integer): number of bytes,
    """

    bytes: int

    def __repr__(self) -> str:


SUMMARY:
**Class Summary: Memory**

The `Memory` class represents a memory allocation with a single field `bytes`. It utilizes the `NamedTuple` data structure.

**Key Features:**

- Single field `bytes` (integer)
- Implements custom `__repr__` method to return human-readable representation of memory size in MB
- Utilizes a separate function `byte_to_mega_bytes` internally

**Implementation Details:**

- Field: `bytes`
- Implementation Method Signature: `str(bytes_to_mega_bytes(self.bytes))`

**Notable Dependencies and Requirements:**

- Reuses the `NamedTuple` data structure from the `typing` module
- Depends on the functionality of the `byte_to_mega_bytes` function (not shown in this snippet)


key: chroma:document
string_value:
CODE:

class MemoryState(NamedTuple):
    """
    `MemoryState` are namedtuples listing frame + CPU/GPU memory with the following fields:

        - `frame` (`Frame`): the current frame (see above)
        - `cpu`: CPU memory consumed at during the current frame as a `Memory` named tuple
        - `gpu`: GPU memory consumed at during the current frame as a `Memory` named tuple
        - `cpu_gpu`: CPU + GPU memory consumed at during the current frame as a `Memory` named tuple
    """

    frame: Frame
    cpu: Memory
    gpu: Memory


SUMMARY:
**Summary**

`MemoryState` class is an immutable namedtuple representing the current memory state, containing:
*   `frame`: the current frame
*   CPU and GPU memory usage (`cpu` and `gpu`) as separate `Memory` tuples
*   A combined `CPU + GPU` memory usage (`cpu_gpu`) tuple.

**Implementation Details**

- Uses an inner class `Frame`, referencing `Memory`, likely used for memory allocation details.
- Utilizes namedtuples from the `typing` module to define and access fields without ambiguity.


key: chroma:document
string_value:
CODE:

class MemorySummary(NamedTuple):
    """
    `MemorySummary` namedtuple otherwise with the fields:

        - `sequential`: a list of `MemoryState` namedtuple (see below) computed from the provided `memory_trace` by
          subtracting the memory after executing each line from the memory before executing said line.
        - `cumulative`: a list of `MemoryState` namedtuple (see below) with cumulative increase in memory for each line
          obtained by summing repeated memory increase for a line if it's executed several times. The list is sorted
          from the frame with the largest memory consumption to the frame with the smallest (can be negative if memory
          is released)
        - `total`: total memory increase during the full tracing as a `Memory` named tuple (see below). Line with
          memory release (negative consumption) are ignored if `ignore_released_memory` is `True` (default).
    """

    sequential: List[MemoryState]
    cumulative: List[MemoryState]
    current: List[MemoryState]


SUMMARY:
**Summary:**

The `MemorySummary` class is a named tuple representing the memory state of a program during tracing. It contains three lists and one total value:

*   **sequential**: A list of `MemoryState` objects, where each represents the memory change after executing a line.
*   **cumulative**: A list of `MemoryState` objects, showing cumulative increases in memory for each line, sorted from highest to lowest consumption.
*   **current**: The current memory state (empty by default).
*   **total**: The total memory increase during tracing, excluding lines with released memory if `ignore_released_memory` is True.

**Key Implementation Details:**

*   Uses `MemoryState`(named tuple) objects to represent the memory changes.
*   Sorts `cumulative` list in descending order by memory consumption.

**Notable Dependencies and Requirements:**

*   Refers to `MemoryState` namedtuple.
*   Assumes existence of a `Memory` named tuple.


