We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Deprecated option `optimize_cuda_cache` warning on import of trl
When you just import trl, we get the following warning:
```
$ python3
Python 3.10.8 (main, Nov 24 2022, 14:13:03) [GCC 11.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import trl
/opt/conda/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.
  warnings.warn(
```

Unless this is left on purpose, I believe that this line https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_config.py#L107, should be changed from:
```
optimize_cuda_cache: bool = False
```
to:
```
optimize_cuda_cache: Optional[bool] = None
```
to get rid of the warning.

I can open a PR if this warning is not needed on default config creation.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `trl/trainer/ppo_config.py`
Content:
```
1 # Copyright 2022 The HuggingFace Team. All rights reserved.
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 #     http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 import json
15 import os
16 import sys
17 import warnings
18 from dataclasses import dataclass, field
19 from typing import Literal, Optional
20 
21 import numpy as np
22 import tyro
23 from typing_extensions import Annotated
24 
25 from trl.trainer.utils import exact_div
26 
27 from ..core import flatten_dict
28 from ..import_utils import is_wandb_available
29 
30 
31 JSONDict = Annotated[Optional[dict], tyro.conf.arg(metavar="JSON", constructor=json.loads)]
32 
33 
34 @dataclass
35 class PPOConfig:
36     """
37     Configuration class for PPOTrainer
38     """
39 
40     # common parameters
41     exp_name: str = os.path.basename(sys.argv[0])[: -len(".py")]
42     """the name of this experiment (by default is the file name without the extension name)"""
43     seed: int = 0
44     """Seed value for random generations"""
45     log_with: Optional[Literal["wandb", "tensorboard"]] = None
46     """Log with either 'wandb' or 'tensorboard', check  https://huggingface.co/docs/accelerate/usage_guides/tracking for more details"""
47     task_name: Optional[str] = None
48     """Name of task to use - used only for tracking purposes"""
49     model_name: Optional[str] = None
50     """Name of model to use - used only for tracking purposes"""
51     query_dataset: Optional[str] = None
52     """Name of dataset to query - used only for tracking purposes"""
53     reward_model: Optional[str] = None
54     """The reward model to use - used only for tracking purposes"""
55     remove_unused_columns: bool = True
56     """Remove unused columns from the dataset if `datasets.Dataset` is used"""
57     tracker_kwargs: JSONDict = field(default_factory=dict)
58     """Keyword arguments for the tracker (e.g. python ppo.py --ppo_config.tracker_kwargs='{"wandb": {"entity": "my_wandb_entity", "name": "my_exp_name"}}'"""
59     accelerator_kwargs: JSONDict = field(default_factory=dict)
60     """Keyword arguments for the accelerator"""
61     project_kwargs: JSONDict = field(default_factory=dict)
62     """Keyword arguments for the accelerator project config (e.g. `logging_dir`)"""
63     tracker_project_name: str = "trl"
64     """Name of project to use for tracking"""
65     push_to_hub_if_best_kwargs: JSONDict = field(default_factory=dict)
66     """Keyword arguments for pushing model to the hub during training (e.g. repo_id)"""
67 
68     # hyperparameters
69     steps: int = 20000
70     """Number of training steps"""
71     learning_rate: float = 1e-5
72     """Adam learning rate"""
73     adap_kl_ctrl: bool = True
74     """Use adaptive KL control, otherwise linear"""
75     init_kl_coef: Optional[float] = 0.2
76     """Initial KL penalty coefficient (used for adaptive and linear control)"""
77     kl_penalty: Literal["kl", "abs", "mse", "full"] = "kl"
78     """kl penalty options: 'kl': model_logp - ref_logp,  'abs': abs(kl),  'mse': mean squared error mse(kl) and 'full': the actual kl for all tokens in the distribution"""
79     target: Optional[float] = 6
80     """Target KL value for adaptive KL control"""
81     horizon: Optional[float] = 10000
82     """Horizon for adaptive KL control"""
83     gamma: float = 1
84     """Gamma parameter for advantage calculation"""
85     lam: float = 0.95
86     """Lambda parameter for advantage calculation"""
87     cliprange: float = 0.2
88     """Range for clipping in PPO policy gradient loss"""
89     cliprange_value: float = 0.2
90     """Range for clipping values in loss calculation"""
91     vf_coef: float = 0.1
92     """Scaling factor for value loss"""
93     batch_size: int = 256
94     """Number of samples per optimisation step"""
95     forward_batch_size: Optional[int] = None
96     """DEPRECATED: use `mini_batch_size` instead, which does the same thing."""
97     mini_batch_size: int = 1
98     """Number of samples optimized in each mini batch"""
99     gradient_accumulation_steps: int = 1
100     """The number of gradient accumulation steps"""
101     world_size: tyro.conf.Suppress[int] = None
102     """The world size for distributed training"""
103     ppo_epochs: int = 4
104     """Number of optimisation epochs per batch of samples"""
105     max_grad_norm: Optional[float] = None
106     """Maximum gradient norm for gradient clipping"""
107     optimize_cuda_cache: bool = False
108     """DEPRECATED: use `optimize_device_cache` instead, which does the same thing."""
109     optimize_device_cache: Optional[bool] = False
110     """Optimize device cache for slightly more memory-efficient training"""
111     early_stopping: bool = False
112     """Whether to stop the PPO optimization loop early is the KL too high"""
113     target_kl: float = 1
114     """Stop early if we exceed this value by over 50%"""
115     compare_steps: int = 1
116     """Number of steps between comparison of the current reward with the best seen so far"""
117     ratio_threshold: float = 10.0
118     """Skip mini-batches with high PPO ratios that can cause loss spikes"""
119     use_score_scaling: bool = False
120     """Use score scaling"""
121     use_score_norm: bool = False
122     """Use score normalization. Only applicable if use_score_scaling is True"""
123     score_clip: Optional[float] = None
124     """Score clipping"""
125     whiten_rewards: bool = False
126     """Whiten the rewards before compute advantages"""
127 
128     # computed hyperparameters at runtime; we use `tyro.conf.Suppress` to hide them from the help text
129     is_encoder_decoder: Optional[tyro.conf.Suppress[bool]] = None
130     """TO BE FILLED In RUNTIME: Whether the model is an encoder-decoder model"""
131     is_peft_model: Optional[tyro.conf.Suppress[bool]] = None
132     """TO BE FILLED In RUNTIME: Whether the model is a PEFT model"""
133     backward_batch_size: tyro.conf.Suppress[int] = None
134     """TO BE FILLED In RUNTIME: Number of samples optimized in an `optimizer.step()` call"""
135     global_backward_batch_size: tyro.conf.Suppress[int] = None
136     """TO BE FILLED In RUNTIME: the effective `backward_batch_size` across all processes"""
137     global_batch_size: tyro.conf.Suppress[int] = None
138     """TO BE FILLED In RUNTIME: the effective `batch_size` across all processes"""
139 
140     if optimize_cuda_cache is not None:
141         warnings.warn(
142             "The `optimize_cuda_cache` argument will be deprecated soon, please use `optimize_device_cache` instead."
143         )
144         optimize_device_cache = optimize_cuda_cache
145     else:
146         optimize_device_cache = False
147 
148     def __post_init__(self):
149         if self.forward_batch_size is not None:
150             warnings.warn(
151                 "Note that using `forward_batch_size` is deprecated, use `mini_batch_size` instead. By setting it you overwrite `mini_batch_size` which affects both the batch size during forward passes and also the mini batch size for PPO optimization."
152             )
153             self.mini_batch_size = self.forward_batch_size
154 
155         self.backward_batch_size = self.mini_batch_size * self.gradient_accumulation_steps
156         exact_div(
157             self.batch_size,
158             self.backward_batch_size,
159             "`batch_size`",
160             "`mini_batch_size * gradient_accumulation_steps`",
161             "`batch_size` must be a multiple of `mini_batch_size * gradient_accumulation_steps`",
162         )
163 
164         # check if wandb is installed
165         if self.log_with == "wandb":
166             # raise error if wandb is not installed
167             if not is_wandb_available():
168                 raise ImportError(
169                     "Please install wandb to use wandb logging. You can do this by running `pip install wandb`."
170                 )
171 
172         self.total_ppo_epochs = int(np.ceil(self.steps / self.batch_size))
173         assert self.kl_penalty in ["kl", "abs", "mse", "full"]
174 
175     def to_dict(self):
176         output_dict = {}
177         for key, value in self.__dict__.items():
178             output_dict[key] = value
179         return flatten_dict(output_dict)
180
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/trl/trainer/ppo_config.py b/trl/trainer/ppo_config.py
--- a/trl/trainer/ppo_config.py
+++ b/trl/trainer/ppo_config.py
@@ -104,7 +104,7 @@
     """Number of optimisation epochs per batch of samples"""
     max_grad_norm: Optional[float] = None
     """Maximum gradient norm for gradient clipping"""
-    optimize_cuda_cache: bool = False
+    optimize_cuda_cache: Optional[bool] = None
     """DEPRECATED: use `optimize_device_cache` instead, which does the same thing."""
     optimize_device_cache: Optional[bool] = False
     """Optimize device cache for slightly more memory-efficient training"""

{"golden_diff": "diff --git a/trl/trainer/ppo_config.py b/trl/trainer/ppo_config.py\n--- a/trl/trainer/ppo_config.py\n+++ b/trl/trainer/ppo_config.py\n@@ -104,7 +104,7 @@\n     \"\"\"Number of optimisation epochs per batch of samples\"\"\"\n     max_grad_norm: Optional[float] = None\n     \"\"\"Maximum gradient norm for gradient clipping\"\"\"\n-    optimize_cuda_cache: bool = False\n+    optimize_cuda_cache: Optional[bool] = None\n     \"\"\"DEPRECATED: use `optimize_device_cache` instead, which does the same thing.\"\"\"\n     optimize_device_cache: Optional[bool] = False\n     \"\"\"Optimize device cache for slightly more memory-efficient training\"\"\"\n", "issue": "Deprecated option `optimize_cuda_cache` warning on import of trl\nWhen you just import trl, we get the following warning:\r\n```\r\n$ python3\r\nPython 3.10.8 (main, Nov 24 2022, 14:13:03) [GCC 11.2.0] on linux\r\nType \"help\", \"copyright\", \"credits\" or \"license\" for more information.\r\n>>> import trl\r\n/opt/conda/lib/python3.10/site-packages/trl/trainer/ppo_config.py:141: UserWarning: The `optimize_cuda_cache` arguement will be deprecated soon, please use `optimize_device_cache` instead.\r\n  warnings.warn(\r\n```\r\n\r\nUnless this is left on purpose, I believe that this line https://github.com/huggingface/trl/blob/main/trl/trainer/ppo_config.py#L107, should be changed from:\r\n```\r\noptimize_cuda_cache: bool = False\r\n```\r\nto:\r\n```\r\noptimize_cuda_cache: Optional[bool] = None\r\n```\r\nto get rid of the warning.\r\n\r\nI can open a PR if this warning is not needed on default config creation.\n", "before_files": [{"content": "# Copyright 2022 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport json\nimport os\nimport sys\nimport warnings\nfrom dataclasses import dataclass, field\nfrom typing import Literal, Optional\n\nimport numpy as np\nimport tyro\nfrom typing_extensions import Annotated\n\nfrom trl.trainer.utils import exact_div\n\nfrom ..core import flatten_dict\nfrom ..import_utils import is_wandb_available\n\n\nJSONDict = Annotated[Optional[dict], tyro.conf.arg(metavar=\"JSON\", constructor=json.loads)]\n\n\n@dataclass\nclass PPOConfig:\n    \"\"\"\n    Configuration class for PPOTrainer\n    \"\"\"\n\n    # common parameters\n    exp_name: str = os.path.basename(sys.argv[0])[: -len(\".py\")]\n    \"\"\"the name of this experiment (by default is the file name without the extension name)\"\"\"\n    seed: int = 0\n    \"\"\"Seed value for random generations\"\"\"\n    log_with: Optional[Literal[\"wandb\", \"tensorboard\"]] = None\n    \"\"\"Log with either 'wandb' or 'tensorboard', check  https://huggingface.co/docs/accelerate/usage_guides/tracking for more details\"\"\"\n    task_name: Optional[str] = None\n    \"\"\"Name of task to use - used only for tracking purposes\"\"\"\n    model_name: Optional[str] = None\n    \"\"\"Name of model to use - used only for tracking purposes\"\"\"\n    query_dataset: Optional[str] = None\n    \"\"\"Name of dataset to query - used only for tracking purposes\"\"\"\n    reward_model: Optional[str] = None\n    \"\"\"The reward model to use - used only for tracking purposes\"\"\"\n    remove_unused_columns: bool = True\n    \"\"\"Remove unused columns from the dataset if `datasets.Dataset` is used\"\"\"\n    tracker_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the tracker (e.g. python ppo.py --ppo_config.tracker_kwargs='{\"wandb\": {\"entity\": \"my_wandb_entity\", \"name\": \"my_exp_name\"}}'\"\"\"\n    accelerator_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the accelerator\"\"\"\n    project_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the accelerator project config (e.g. `logging_dir`)\"\"\"\n    tracker_project_name: str = \"trl\"\n    \"\"\"Name of project to use for tracking\"\"\"\n    push_to_hub_if_best_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for pushing model to the hub during training (e.g. repo_id)\"\"\"\n\n    # hyperparameters\n    steps: int = 20000\n    \"\"\"Number of training steps\"\"\"\n    learning_rate: float = 1e-5\n    \"\"\"Adam learning rate\"\"\"\n    adap_kl_ctrl: bool = True\n    \"\"\"Use adaptive KL control, otherwise linear\"\"\"\n    init_kl_coef: Optional[float] = 0.2\n    \"\"\"Initial KL penalty coefficient (used for adaptive and linear control)\"\"\"\n    kl_penalty: Literal[\"kl\", \"abs\", \"mse\", \"full\"] = \"kl\"\n    \"\"\"kl penalty options: 'kl': model_logp - ref_logp,  'abs': abs(kl),  'mse': mean squared error mse(kl) and 'full': the actual kl for all tokens in the distribution\"\"\"\n    target: Optional[float] = 6\n    \"\"\"Target KL value for adaptive KL control\"\"\"\n    horizon: Optional[float] = 10000\n    \"\"\"Horizon for adaptive KL control\"\"\"\n    gamma: float = 1\n    \"\"\"Gamma parameter for advantage calculation\"\"\"\n    lam: float = 0.95\n    \"\"\"Lambda parameter for advantage calculation\"\"\"\n    cliprange: float = 0.2\n    \"\"\"Range for clipping in PPO policy gradient loss\"\"\"\n    cliprange_value: float = 0.2\n    \"\"\"Range for clipping values in loss calculation\"\"\"\n    vf_coef: float = 0.1\n    \"\"\"Scaling factor for value loss\"\"\"\n    batch_size: int = 256\n    \"\"\"Number of samples per optimisation step\"\"\"\n    forward_batch_size: Optional[int] = None\n    \"\"\"DEPRECATED: use `mini_batch_size` instead, which does the same thing.\"\"\"\n    mini_batch_size: int = 1\n    \"\"\"Number of samples optimized in each mini batch\"\"\"\n    gradient_accumulation_steps: int = 1\n    \"\"\"The number of gradient accumulation steps\"\"\"\n    world_size: tyro.conf.Suppress[int] = None\n    \"\"\"The world size for distributed training\"\"\"\n    ppo_epochs: int = 4\n    \"\"\"Number of optimisation epochs per batch of samples\"\"\"\n    max_grad_norm: Optional[float] = None\n    \"\"\"Maximum gradient norm for gradient clipping\"\"\"\n    optimize_cuda_cache: bool = False\n    \"\"\"DEPRECATED: use `optimize_device_cache` instead, which does the same thing.\"\"\"\n    optimize_device_cache: Optional[bool] = False\n    \"\"\"Optimize device cache for slightly more memory-efficient training\"\"\"\n    early_stopping: bool = False\n    \"\"\"Whether to stop the PPO optimization loop early is the KL too high\"\"\"\n    target_kl: float = 1\n    \"\"\"Stop early if we exceed this value by over 50%\"\"\"\n    compare_steps: int = 1\n    \"\"\"Number of steps between comparison of the current reward with the best seen so far\"\"\"\n    ratio_threshold: float = 10.0\n    \"\"\"Skip mini-batches with high PPO ratios that can cause loss spikes\"\"\"\n    use_score_scaling: bool = False\n    \"\"\"Use score scaling\"\"\"\n    use_score_norm: bool = False\n    \"\"\"Use score normalization. Only applicable if use_score_scaling is True\"\"\"\n    score_clip: Optional[float] = None\n    \"\"\"Score clipping\"\"\"\n    whiten_rewards: bool = False\n    \"\"\"Whiten the rewards before compute advantages\"\"\"\n\n    # computed hyperparameters at runtime; we use `tyro.conf.Suppress` to hide them from the help text\n    is_encoder_decoder: Optional[tyro.conf.Suppress[bool]] = None\n    \"\"\"TO BE FILLED In RUNTIME: Whether the model is an encoder-decoder model\"\"\"\n    is_peft_model: Optional[tyro.conf.Suppress[bool]] = None\n    \"\"\"TO BE FILLED In RUNTIME: Whether the model is a PEFT model\"\"\"\n    backward_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: Number of samples optimized in an `optimizer.step()` call\"\"\"\n    global_backward_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: the effective `backward_batch_size` across all processes\"\"\"\n    global_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: the effective `batch_size` across all processes\"\"\"\n\n    if optimize_cuda_cache is not None:\n        warnings.warn(\n            \"The `optimize_cuda_cache` argument will be deprecated soon, please use `optimize_device_cache` instead.\"\n        )\n        optimize_device_cache = optimize_cuda_cache\n    else:\n        optimize_device_cache = False\n\n    def __post_init__(self):\n        if self.forward_batch_size is not None:\n            warnings.warn(\n                \"Note that using `forward_batch_size` is deprecated, use `mini_batch_size` instead. By setting it you overwrite `mini_batch_size` which affects both the batch size during forward passes and also the mini batch size for PPO optimization.\"\n            )\n            self.mini_batch_size = self.forward_batch_size\n\n        self.backward_batch_size = self.mini_batch_size * self.gradient_accumulation_steps\n        exact_div(\n            self.batch_size,\n            self.backward_batch_size,\n            \"`batch_size`\",\n            \"`mini_batch_size * gradient_accumulation_steps`\",\n            \"`batch_size` must be a multiple of `mini_batch_size * gradient_accumulation_steps`\",\n        )\n\n        # check if wandb is installed\n        if self.log_with == \"wandb\":\n            # raise error if wandb is not installed\n            if not is_wandb_available():\n                raise ImportError(\n                    \"Please install wandb to use wandb logging. You can do this by running `pip install wandb`.\"\n                )\n\n        self.total_ppo_epochs = int(np.ceil(self.steps / self.batch_size))\n        assert self.kl_penalty in [\"kl\", \"abs\", \"mse\", \"full\"]\n\n    def to_dict(self):\n        output_dict = {}\n        for key, value in self.__dict__.items():\n            output_dict[key] = value\n        return flatten_dict(output_dict)\n", "path": "trl/trainer/ppo_config.py"}], "after_files": [{"content": "# Copyright 2022 The HuggingFace Team. All rights reserved.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport json\nimport os\nimport sys\nimport warnings\nfrom dataclasses import dataclass, field\nfrom typing import Literal, Optional\n\nimport numpy as np\nimport tyro\nfrom typing_extensions import Annotated\n\nfrom trl.trainer.utils import exact_div\n\nfrom ..core import flatten_dict\nfrom ..import_utils import is_wandb_available\n\n\nJSONDict = Annotated[Optional[dict], tyro.conf.arg(metavar=\"JSON\", constructor=json.loads)]\n\n\n@dataclass\nclass PPOConfig:\n    \"\"\"\n    Configuration class for PPOTrainer\n    \"\"\"\n\n    # common parameters\n    exp_name: str = os.path.basename(sys.argv[0])[: -len(\".py\")]\n    \"\"\"the name of this experiment (by default is the file name without the extension name)\"\"\"\n    seed: int = 0\n    \"\"\"Seed value for random generations\"\"\"\n    log_with: Optional[Literal[\"wandb\", \"tensorboard\"]] = None\n    \"\"\"Log with either 'wandb' or 'tensorboard', check  https://huggingface.co/docs/accelerate/usage_guides/tracking for more details\"\"\"\n    task_name: Optional[str] = None\n    \"\"\"Name of task to use - used only for tracking purposes\"\"\"\n    model_name: Optional[str] = None\n    \"\"\"Name of model to use - used only for tracking purposes\"\"\"\n    query_dataset: Optional[str] = None\n    \"\"\"Name of dataset to query - used only for tracking purposes\"\"\"\n    reward_model: Optional[str] = None\n    \"\"\"The reward model to use - used only for tracking purposes\"\"\"\n    remove_unused_columns: bool = True\n    \"\"\"Remove unused columns from the dataset if `datasets.Dataset` is used\"\"\"\n    tracker_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the tracker (e.g. python ppo.py --ppo_config.tracker_kwargs='{\"wandb\": {\"entity\": \"my_wandb_entity\", \"name\": \"my_exp_name\"}}'\"\"\"\n    accelerator_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the accelerator\"\"\"\n    project_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for the accelerator project config (e.g. `logging_dir`)\"\"\"\n    tracker_project_name: str = \"trl\"\n    \"\"\"Name of project to use for tracking\"\"\"\n    push_to_hub_if_best_kwargs: JSONDict = field(default_factory=dict)\n    \"\"\"Keyword arguments for pushing model to the hub during training (e.g. repo_id)\"\"\"\n\n    # hyperparameters\n    steps: int = 20000\n    \"\"\"Number of training steps\"\"\"\n    learning_rate: float = 1e-5\n    \"\"\"Adam learning rate\"\"\"\n    adap_kl_ctrl: bool = True\n    \"\"\"Use adaptive KL control, otherwise linear\"\"\"\n    init_kl_coef: Optional[float] = 0.2\n    \"\"\"Initial KL penalty coefficient (used for adaptive and linear control)\"\"\"\n    kl_penalty: Literal[\"kl\", \"abs\", \"mse\", \"full\"] = \"kl\"\n    \"\"\"kl penalty options: 'kl': model_logp - ref_logp,  'abs': abs(kl),  'mse': mean squared error mse(kl) and 'full': the actual kl for all tokens in the distribution\"\"\"\n    target: Optional[float] = 6\n    \"\"\"Target KL value for adaptive KL control\"\"\"\n    horizon: Optional[float] = 10000\n    \"\"\"Horizon for adaptive KL control\"\"\"\n    gamma: float = 1\n    \"\"\"Gamma parameter for advantage calculation\"\"\"\n    lam: float = 0.95\n    \"\"\"Lambda parameter for advantage calculation\"\"\"\n    cliprange: float = 0.2\n    \"\"\"Range for clipping in PPO policy gradient loss\"\"\"\n    cliprange_value: float = 0.2\n    \"\"\"Range for clipping values in loss calculation\"\"\"\n    vf_coef: float = 0.1\n    \"\"\"Scaling factor for value loss\"\"\"\n    batch_size: int = 256\n    \"\"\"Number of samples per optimisation step\"\"\"\n    forward_batch_size: Optional[int] = None\n    \"\"\"DEPRECATED: use `mini_batch_size` instead, which does the same thing.\"\"\"\n    mini_batch_size: int = 1\n    \"\"\"Number of samples optimized in each mini batch\"\"\"\n    gradient_accumulation_steps: int = 1\n    \"\"\"The number of gradient accumulation steps\"\"\"\n    world_size: tyro.conf.Suppress[int] = None\n    \"\"\"The world size for distributed training\"\"\"\n    ppo_epochs: int = 4\n    \"\"\"Number of optimisation epochs per batch of samples\"\"\"\n    max_grad_norm: Optional[float] = None\n    \"\"\"Maximum gradient norm for gradient clipping\"\"\"\n    optimize_cuda_cache: Optional[bool] = None\n    \"\"\"DEPRECATED: use `optimize_device_cache` instead, which does the same thing.\"\"\"\n    optimize_device_cache: Optional[bool] = False\n    \"\"\"Optimize device cache for slightly more memory-efficient training\"\"\"\n    early_stopping: bool = False\n    \"\"\"Whether to stop the PPO optimization loop early is the KL too high\"\"\"\n    target_kl: float = 1\n    \"\"\"Stop early if we exceed this value by over 50%\"\"\"\n    compare_steps: int = 1\n    \"\"\"Number of steps between comparison of the current reward with the best seen so far\"\"\"\n    ratio_threshold: float = 10.0\n    \"\"\"Skip mini-batches with high PPO ratios that can cause loss spikes\"\"\"\n    use_score_scaling: bool = False\n    \"\"\"Use score scaling\"\"\"\n    use_score_norm: bool = False\n    \"\"\"Use score normalization. Only applicable if use_score_scaling is True\"\"\"\n    score_clip: Optional[float] = None\n    \"\"\"Score clipping\"\"\"\n    whiten_rewards: bool = False\n    \"\"\"Whiten the rewards before compute advantages\"\"\"\n\n    # computed hyperparameters at runtime; we use `tyro.conf.Suppress` to hide them from the help text\n    is_encoder_decoder: Optional[tyro.conf.Suppress[bool]] = None\n    \"\"\"TO BE FILLED In RUNTIME: Whether the model is an encoder-decoder model\"\"\"\n    is_peft_model: Optional[tyro.conf.Suppress[bool]] = None\n    \"\"\"TO BE FILLED In RUNTIME: Whether the model is a PEFT model\"\"\"\n    backward_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: Number of samples optimized in an `optimizer.step()` call\"\"\"\n    global_backward_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: the effective `backward_batch_size` across all processes\"\"\"\n    global_batch_size: tyro.conf.Suppress[int] = None\n    \"\"\"TO BE FILLED In RUNTIME: the effective `batch_size` across all processes\"\"\"\n\n    if optimize_cuda_cache is not None:\n        warnings.warn(\n            \"The `optimize_cuda_cache` argument will be deprecated soon, please use `optimize_device_cache` instead.\"\n        )\n        optimize_device_cache = optimize_cuda_cache\n    else:\n        optimize_device_cache = False\n\n    def __post_init__(self):\n        if self.forward_batch_size is not None:\n            warnings.warn(\n                \"Note that using `forward_batch_size` is deprecated, use `mini_batch_size` instead. By setting it you overwrite `mini_batch_size` which affects both the batch size during forward passes and also the mini batch size for PPO optimization.\"\n            )\n            self.mini_batch_size = self.forward_batch_size\n\n        self.backward_batch_size = self.mini_batch_size * self.gradient_accumulation_steps\n        exact_div(\n            self.batch_size,\n            self.backward_batch_size,\n            \"`batch_size`\",\n            \"`mini_batch_size * gradient_accumulation_steps`\",\n            \"`batch_size` must be a multiple of `mini_batch_size * gradient_accumulation_steps`\",\n        )\n\n        # check if wandb is installed\n        if self.log_with == \"wandb\":\n            # raise error if wandb is not installed\n            if not is_wandb_available():\n                raise ImportError(\n                    \"Please install wandb to use wandb logging. You can do this by running `pip install wandb`.\"\n                )\n\n        self.total_ppo_epochs = int(np.ceil(self.steps / self.batch_size))\n        assert self.kl_penalty in [\"kl\", \"abs\", \"mse\", \"full\"]\n\n    def to_dict(self):\n        output_dict = {}\n        for key, value in self.__dict__.items():\n            output_dict[key] = value\n        return flatten_dict(output_dict)\n", "path": "trl/trainer/ppo_config.py"}]}