# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2aae18be-3c83-40ce-a8d9-519140821e86
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/clis", "title": "Command Line Interfaces (CLIs)", "tokens": 1209, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2aae18be-3c83-40ce-a8d9-519140821e86\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/clis\", \"title\": \"Command Line Interfaces (CLIs)\", \"tokens\": 1209, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b83b6555-15cb-4b57-8077-36c3568166a4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/clis\", \"title\": \"Command Line Interfaces (CLIs)\", \"tokens\": 1209, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3c7b62a48b18c3b8de67f9221dc33fa6352bc10f7d42d3a454483bcde4049576\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d66c8401-3e52-48a7-a817-89883443903f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"112597e2d60298bd753e703a67df805412248ae7c8fa48ceabe2d7a64a647f7a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3056, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "doc_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "ref_doc_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "chroma:document": "# Command Line Interfaces (CLIs)\n\nYou can use TRL to fine-tune your Language Model with Supervised Fine-Tuning (SFT) or Direct Policy Optimization (DPO) or even chat with your model using the TRL CLIs.\n\nCurrently supported CLIs are:\n\n- `trl sft`: fine-tune a LLM on a text/instruction dataset\n- `trl dpo`: fine-tune a LLM with DPO on a preference dataset \n- `trl chat`: quickly spin up a LLM fine-tuned for chatting\n\n## Fine-tuning with the CLI\n\nBefore getting started, pick up a Language Model from Hugging Face Hub. Supported models can be found with the filter \"text-generation\" within models. Also make sure to pick up a relevant dataset for your task.\n\nBefore using the `sft` or `dpo` commands make sure to run:\n```bash\naccelerate config\n```\nand pick up the right configuration for your training setup (single / multi-GPU, DeepSpeed, etc.). Make sure to complete all steps of `accelerate config` before running any CLI command.\n\nWe also recommend you passing a YAML config file to configure your training protocol. Below is a simple example of a YAML file that you can use for training your models with `trl sft` command.\n\n```yaml\nmodel_name_or_path:\n  trl-internal-testing/tiny-random-LlamaForCausalLM\ndataset_name:\n  imdb\ndataset_text_field:\n  text\nreport_to:\n  none\nlearning_rate:\n  0.0001\nlr_scheduler_type:\n  cosine\n```\n\nSave that config in a `.yaml` and get started immediately! An example CLI config is available as `examples/cli_configs/example_config.yaml`. Note you can overwrite the arguments from the config file by explicitly passing them to the CLI, e.g. from the root folder:\n\n```bash\ntrl sft --config examples/cli_configs/example_config.yaml --output_dir test-trl-cli --lr_scheduler_type cosine_with_restarts\n```\n\nWill force-use `cosine_with_restarts` for `lr_scheduler_type`.\n\n### Supported Arguments \n\nWe do support all arguments from `transformers.TrainingArguments`, for loading your model, we support all arguments from `~trl.ModelConfig`:\n\n[[autodoc]] ModelConfig\n\nYou can pass any of these arguments either to the CLI or the YAML file.\n\n### Supervised Fine-tuning (SFT)\n\nFollow the basic instructions above and run `trl sft --output_dir <output_dir> <*args>`: \n\n```bash\ntrl sft --model_name_or_path facebook/opt-125m --dataset_name imdb --output_dir opt-sft-imdb\n```\n\nThe SFT CLI is based on the `examples/scripts/sft.py` script.\n\n### Direct Policy Optimization (DPO)\n\nTo use the DPO CLI, you need to have a dataset in the TRL format such as \n\n* TRL's Anthropic HH dataset: https://huggingface.co/datasets/trl-internal-testing/hh-rlhf-helpful-base-trl-style\n* TRL's OpenAI TL;DR summarization dataset: https://huggingface.co/datasets/trl-internal-testing/tldr-preference-trl-style\n\nThese datasets always have at least three columns `prompt, chosen, rejected`:\n\n* `prompt` is a list of strings.\n* `chosen` is the chosen response in [chat format](https://huggingface.co/docs/transformers/main/en/chat_templating)\n* `rejected` is the rejected response [chat format](https://huggingface.co/docs/transformers/main/en/chat_templating)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d66c8401-3e52-48a7-a817-89883443903f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/clis", "title": "Command Line Interfaces (CLIs)", "tokens": 1209, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d66c8401-3e52-48a7-a817-89883443903f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/clis\", \"title\": \"Command Line Interfaces (CLIs)\", \"tokens\": 1209, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b83b6555-15cb-4b57-8077-36c3568166a4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/clis\", \"title\": \"Command Line Interfaces (CLIs)\", \"tokens\": 1209, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3c7b62a48b18c3b8de67f9221dc33fa6352bc10f7d42d3a454483bcde4049576\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2aae18be-3c83-40ce-a8d9-519140821e86\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/clis\", \"title\": \"Command Line Interfaces (CLIs)\", \"tokens\": 1209, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4c076177e2125357a7fd9022e10cb05ca7b3c2a95beea33c83aaed00a75394b7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3060, \"end_char_idx\": 4924, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "doc_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "ref_doc_id": "b83b6555-15cb-4b57-8077-36c3568166a4", "chroma:document": "To do a quick start, you can run the following command:\n\n```bash\ntrl dpo --model_name_or_path facebook/opt-125m --output_dir trl-hh-rlhf --dataset_name trl-internal-testing/hh-rlhf-helpful-base-trl-style\n```\n\n\nThe DPO CLI is based on the `examples/scripts/dpo.py` script.\n\n\n#### Custom preference dataset\n\nFormat the dataset into TRL format (you can adapt the `examples/datasets/anthropic_hh.py`):\n\n```bash\npython examples/datasets/anthropic_hh.py --push_to_hub --hf_entity your-hf-org\n```\n\n## Chat interface\n\nThe chat CLI lets you quickly load the model and talk to it. Simply run the following:\n\n```bash\ntrl chat --model_name_or_path  Qwen/Qwen1.5-0.5B-Chat \n```\n\n> [!TIP]\n> To use the chat CLI with the developer installation, you must run `make dev` \n>\n\nNote that the chat interface relies on the tokenizer's [chat template](https://huggingface.co/docs/transformers/chat_templating) to format the inputs for the model. Make sure your tokenizer has a chat template defined.\n\nBesides talking to the model there are a few commands you can use:\n\n- **clear**: clears the current conversation and start a new one\n- **example {NAME}**: load example named `{NAME}` from the config and use it as the user input\n- **set {SETTING_NAME}={SETTING_VALUE};**: change the system prompt or generation settings (multiple settings are separated by a ';').\n- **reset**: same as clear but also resets the generation configs to defaults if they have been changed by **set**\n- **save {SAVE_NAME} (optional)**: save the current chat and settings to file by default to `./chat_history/{MODEL_NAME}/chat_{DATETIME}.yaml` or `{SAVE_NAME}` if provided\n- **exit**: closes the interface\n\nThe default examples are defined in `examples/scripts/config/default_chat_config.yaml` but you can pass your own with `--config CONFIG_FILE` where you can also specify the default generation parameters."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 227192cc-5be8-44e4-a048-71bc5e53b1e4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"227192cc-5be8-44e4-a048-71bc5e53b1e4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d10eb15b-17af-436d-b591-c925fce94950\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d3caf8f18dca2e3da6fc656dcffb09e29a1d8c51c9c11afec139d4067311bc6d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2811, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "# Online DPO Trainer\n\nTRL supports training LLMs with online DPO ([Guo et al., 2024](https://huggingface.co/papers/2402.04792)) with a reward model (RM). The idea of online DPO is to generate completions based on prompts and either have a reward model or an LLM judge to rank the responses as chosen or rejected. Then the model is updated with the ranked responses using the DPO loss.\n\nWhile [Guo et al. (2024)](https://huggingface.co/papers/2402.04792) used an LLM judge to score model completions, the current implementation only supports reward models -- see [Reward Bench](https://huggingface.co/spaces/allenai/reward-bench) for a leaderboard of public models you can use.\n\n## Get started\n\nThe basic API looks as follows:\n\n```python\nfrom datasets import Dataset\nfrom trl import OnlineDPOConfig, OnlineDPOTrainer\nfrom transformers import (\n    AutoModelForCausalLM,\n    AutoModelForSequenceClassification,\n    AutoTokenizer,\n)\nNUM_DUMMY_SAMPLES = 100\ntokenizer = AutoTokenizer.from_pretrained(\"HuggingFaceTB/SmolLM-135M-Instruct\")\ntok.add_special_tokens({\"pad_token\": \"[PAD]\"})\n# The model to optimise\nmodel = AutoModelForCausalLM.from_pretrained(\"HuggingFaceTB/SmolLM-135M-Instruct\")\n# The reference model to calculate the KL divergence against\nref_model = AutoModelForCausalLM.from_pretrained(\"HuggingFaceTB/SmolLM-135M-Instruct\")\n# The model to score completions with. In practice, you will need a fine-tuned reward model.\nreward_model = AutoModelForSequenceClassification.from_pretrained(\"HuggingFaceTB/SmolLM-135M-Instruct\", num_labels=1)\ntrain_dataset = Dataset.from_dict(\n    {\"input_ids\": [tok.encode(\"Q: Hi how are you? A:\")] * NUM_DUMMY_SAMPLES})\neval_dataset = Dataset.from_dict(\n    {\"input_ids\": [tok.encode(\"Q: What do you like to eat A:\")] * NUM_DUMMY_SAMPLES})\ntrainer = OnlineDPOTrainer(\n    OnlineDPOConfig(\n        output_dir=\"online-dpo-model\",\n    ),\n    model=model,\n    ref_model=ref_model,\n    reward_model=reward_model,\n    tokenizer=tok,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n)\ntrainer.train()\n```\n\nTo run the online DPO script with a dummy reward model, run:\n\n```bash\npython examples/scripts/online_dpo.py \\\n    --dataset_name trl-lib/tldr \\\n    --learning_rate 3e-6 \\\n    --output_dir models/minimal/online_dpo \\\n    --per_device_train_batch_size 1 \\\n    --gradient_accumulation_steps 64 \\\n    --total_episodes 30000 \\\n    --model_name_or_path EleutherAI/pythia-14m \\\n    --sft_model_path EleutherAI/pythia-14m \\\n    --reward_model_path EleutherAI/pythia-14m \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n    --response_length 53 \\\n    --sanity_check\n```\n\n## Expected dataset format\n\nUnlike standard DPO where one provides a dataset with chosen and rejected columns, for online DPO one just needs a dataset of prompts to generate the completions from."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d10eb15b-17af-436d-b591-c925fce94950
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d10eb15b-17af-436d-b591-c925fce94950\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"227192cc-5be8-44e4-a048-71bc5e53b1e4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"7d3ba5402064537c148b28de94d9838fd4c5e05197a7202d1758635065c90f0d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b4e898a9-b92a-4697-9730-ccbaa5c361cd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"38689ed1b054a9b0e970f7261aafcce3841a7cad2d113d0caed7c5f7ba172008\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1375, \"end_char_idx\": 4546, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "In practice, you will need a fine-tuned reward model.\nreward_model = AutoModelForSequenceClassification.from_pretrained(\"HuggingFaceTB/SmolLM-135M-Instruct\", num_labels=1)\ntrain_dataset = Dataset.from_dict(\n    {\"input_ids\": [tok.encode(\"Q: Hi how are you? A:\")] * NUM_DUMMY_SAMPLES})\neval_dataset = Dataset.from_dict(\n    {\"input_ids\": [tok.encode(\"Q: What do you like to eat A:\")] * NUM_DUMMY_SAMPLES})\ntrainer = OnlineDPOTrainer(\n    OnlineDPOConfig(\n        output_dir=\"online-dpo-model\",\n    ),\n    model=model,\n    ref_model=ref_model,\n    reward_model=reward_model,\n    tokenizer=tok,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n)\ntrainer.train()\n```\n\nTo run the online DPO script with a dummy reward model, run:\n\n```bash\npython examples/scripts/online_dpo.py \\\n    --dataset_name trl-lib/tldr \\\n    --learning_rate 3e-6 \\\n    --output_dir models/minimal/online_dpo \\\n    --per_device_train_batch_size 1 \\\n    --gradient_accumulation_steps 64 \\\n    --total_episodes 30000 \\\n    --model_name_or_path EleutherAI/pythia-14m \\\n    --sft_model_path EleutherAI/pythia-14m \\\n    --reward_model_path EleutherAI/pythia-14m \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n    --response_length 53 \\\n    --sanity_check\n```\n\n## Expected dataset format\n\nUnlike standard DPO where one provides a dataset with chosen and rejected columns, for online DPO one just needs a dataset of prompts to generate the completions from. The [`OnlineDPOTrainer`] assumes that the dataset is preprocessed for model inference, so typically you will want to wrap your prompts in the messages format and then apply the chat template as follows:\n\n```python\ndef prepare_dataset(dataset, tokenizer, dataset_prompt_field):\n    \"\"\"pre-tokenize the dataset before training; only collate during training\"\"\"\n    return dataset.map(\n        lambda x: {\"input_ids\": tokenizer.apply_chat_template(x[dataset_prompt_field], add_generation_prompt=True)},\n        remove_columns=dataset.column_names,\n    )\n\ndataset = prepare_dataset(dataset)\n```\n\n## Explanation of the logged metrics\n\nThe logged metrics are as follows. Here is an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35)\n\n* `eps`: Tracks the number of episodes per second.\n* `objective/kl`: The mean Kullback-Leibler (KL) divergence between the current model and reference model.\n* `objective/entropy`: The mean entropy of the model, indicating the randomness of the actions chosen by the model.\n* `objective/non_score_reward`: The mean reward from non-score-related sources, basically `beta * kl.sum(1)`, where `beta` is the KL penalty coefficient and `kl` is the per-token KL divergence.\n* `objective/rlhf_reward`: The mean RLHF reward, which is `score - non_score_reward`.\n* `objective/scores`: The mean scores returned by the reward model / environment.\n* `objective/scores_margin`: The mean score margin (according to the external reward model) between the chosen and rejected completions.\n* `rewards/accuracies`: The accuracies of the online DPO's implicit reward model.\n* `rewards/chosen`: The mean reward (according to online DPO's implicit reward model)of the chosen completions."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b4e898a9-b92a-4697-9730-ccbaa5c361cd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b4e898a9-b92a-4697-9730-ccbaa5c361cd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d10eb15b-17af-436d-b591-c925fce94950\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3acbbbea5611923e7cf5d87042fb083409f9d3bcef70ec7fe3dc4aef1dd52525\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3a6a7e25-53a2-4c0c-aa4f-0cdfff914803\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b29224f9019a23e1199a1fb59a586f2fccef67847212f26d9933970924d9acb5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2812, \"end_char_idx\": 6244, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "The [`OnlineDPOTrainer`] assumes that the dataset is preprocessed for model inference, so typically you will want to wrap your prompts in the messages format and then apply the chat template as follows:\n\n```python\ndef prepare_dataset(dataset, tokenizer, dataset_prompt_field):\n    \"\"\"pre-tokenize the dataset before training; only collate during training\"\"\"\n    return dataset.map(\n        lambda x: {\"input_ids\": tokenizer.apply_chat_template(x[dataset_prompt_field], add_generation_prompt=True)},\n        remove_columns=dataset.column_names,\n    )\n\ndataset = prepare_dataset(dataset)\n```\n\n## Explanation of the logged metrics\n\nThe logged metrics are as follows. Here is an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35)\n\n* `eps`: Tracks the number of episodes per second.\n* `objective/kl`: The mean Kullback-Leibler (KL) divergence between the current model and reference model.\n* `objective/entropy`: The mean entropy of the model, indicating the randomness of the actions chosen by the model.\n* `objective/non_score_reward`: The mean reward from non-score-related sources, basically `beta * kl.sum(1)`, where `beta` is the KL penalty coefficient and `kl` is the per-token KL divergence.\n* `objective/rlhf_reward`: The mean RLHF reward, which is `score - non_score_reward`.\n* `objective/scores`: The mean scores returned by the reward model / environment.\n* `objective/scores_margin`: The mean score margin (according to the external reward model) between the chosen and rejected completions.\n* `rewards/accuracies`: The accuracies of the online DPO's implicit reward model.\n* `rewards/chosen`: The mean reward (according to online DPO's implicit reward model)of the chosen completions.\n* `rewards/rejected`: The mean reward (according to online DPO's implicit reward model) of the rejected completions.\n* `rewards/margins`: The mean reward margin (according to online DPO's implicit reward model) between the chosen and rejected completions.\n* `logps/chosen`: The mean log probabilities of the chosen completions.\n* `logps/rejected`: The mean log probabilities of the rejected completions.\n* `val/num_eos_tokens`: The number of end-of-sequence (EOS) tokens generated, which can indicate the number of complete responses.\n* `lr`: lr: The current learning rate used by the optimizer.\n* `episode`: episode: The current global step or episode count in the training process.\n\n\n## Cookbook\n\n> [!IMPORTANT]\n> Make sure the SFT model and reward model use the _same_ chat template. Otherwise you may find the model completions are scored incorrectly.\n\n\n* Debugging TIP: `objective/rlhf_reward`: this is the ultimate objective of the RLHF training. If training works as intended, this metric should keep going up.\n* Memory TIP: If you are running out of memory, you can try to reduce the `--per_device_train_batch_size` or increase the `--gradient_accumulation_steps` to reduce the memory footprint.\n* Memory TIP: If you have multiple GPUs, you can also run training with DeepSpeed stage 3 to reduce the memory footprint `accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml`.\n* Usage TIP: We recommend to use the \"EOS trick\" via `--non_eos_penalty --stop_token eos`, which replaces the score of completions that do not end with an EOS token with a static scalar penalty `--penalty_reward_value`. This can help the model learn to generate more coherent completions."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3a6a7e25-53a2-4c0c-aa4f-0cdfff914803
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3a6a7e25-53a2-4c0c-aa4f-0cdfff914803\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b4e898a9-b92a-4697-9730-ccbaa5c361cd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"bdcd68cf7c2de2c8e57c752e3263361301bd519e69146519d510fcd381afc58d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"68fa7d05-b2a0-4295-9dbf-1abca1b5e99f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"76840db126c61286971ede7b1eafe76ebd6bba9010baa21d2cfd212e151519e8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4547, \"end_char_idx\": 7424, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "* `rewards/rejected`: The mean reward (according to online DPO's implicit reward model) of the rejected completions.\n* `rewards/margins`: The mean reward margin (according to online DPO's implicit reward model) between the chosen and rejected completions.\n* `logps/chosen`: The mean log probabilities of the chosen completions.\n* `logps/rejected`: The mean log probabilities of the rejected completions.\n* `val/num_eos_tokens`: The number of end-of-sequence (EOS) tokens generated, which can indicate the number of complete responses.\n* `lr`: lr: The current learning rate used by the optimizer.\n* `episode`: episode: The current global step or episode count in the training process.\n\n\n## Cookbook\n\n> [!IMPORTANT]\n> Make sure the SFT model and reward model use the _same_ chat template. Otherwise you may find the model completions are scored incorrectly.\n\n\n* Debugging TIP: `objective/rlhf_reward`: this is the ultimate objective of the RLHF training. If training works as intended, this metric should keep going up.\n* Memory TIP: If you are running out of memory, you can try to reduce the `--per_device_train_batch_size` or increase the `--gradient_accumulation_steps` to reduce the memory footprint.\n* Memory TIP: If you have multiple GPUs, you can also run training with DeepSpeed stage 3 to reduce the memory footprint `accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml`.\n* Usage TIP: We recommend to use the \"EOS trick\" via `--non_eos_penalty --stop_token eos`, which replaces the score of completions that do not end with an EOS token with a static scalar penalty `--penalty_reward_value`. This can help the model learn to generate more coherent completions.\n\n\n## What is my model doing exactly?\n\nTo help you understand what your model is doing, we periodically log some sample completions from the model. Here is an example of a completion. In an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35), it looks like the following, allowing you to see the model's response at different stages of training. By default we generate `--num_sample_generations 10` during training, but you can customize the number of generations.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/ppov2_completions.gif)\n\n\nIn the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 68fa7d05-b2a0-4295-9dbf-1abca1b5e99f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"68fa7d05-b2a0-4295-9dbf-1abca1b5e99f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3a6a7e25-53a2-4c0c-aa4f-0cdfff914803\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cdb85b896c7ac7e7d4618d78c798de75156ea836011afbb1ad66218b1f4c6b7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52f7ab45-2a74-4cc6-adb3-a00d8461741f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d17b65a0a505a96450e9e80ee9f124305aa689b75da7efe337bff523ba019494\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6247, \"end_char_idx\": 9849, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "## What is my model doing exactly?\n\nTo help you understand what your model is doing, we periodically log some sample completions from the model. Here is an example of a completion. In an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35), it looks like the following, allowing you to see the model's response at different stages of training. By default we generate `--num_sample_generations 10` during training, but you can customize the number of generations.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/ppov2_completions.gif)\n\n\nIn the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?               \u2502 desperate.<|endoftext|>[PAD][P\u2026 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 POST: Hi,                       \u2502                                 \u2502          \u2502\n\u2502 I'm 22, and I have been with my \u2502                                 \u2502          \u2502\n\u2502 girlfriend for 5 years now. We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Problem, I recently started to  \u2502                                 \u2502          \u2502\n\u2502 have feelings for an other      \u2502                                 \u2502          \u2502\n\u2502 person (a friend). This person  \u2502                                 \u2502          \u2502\n\u2502 has had a boyfriend for now 3   \u2502                                 \u2502          \u2502\n\u2502 years, and has absolutely no    \u2502                                 \u2502          \u2502\n\u2502 ideas. Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 52f7ab45-2a74-4cc6-adb3-a00d8461741f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"52f7ab45-2a74-4cc6-adb3-a00d8461741f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"68fa7d05-b2a0-4295-9dbf-1abca1b5e99f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"db8e8239c468b4becbdb5000d9c2ed05752b21d4a7dcc8a371c6173e84eeb5ad\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"78f28d4e-be0c-4b0e-a1a1-acf08ede9b06\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6881618b741a14aab65976a68647bfc5d066005398f47fab6a037a4a84cc3f06\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7759, \"end_char_idx\": 11481, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Problem, I recently started to  \u2502                                 \u2502          \u2502\n\u2502 have feelings for an other      \u2502                                 \u2502          \u2502\n\u2502 person (a friend). This person  \u2502                                 \u2502          \u2502\n\u2502 has had a boyfriend for now 3   \u2502                                 \u2502          \u2502\n\u2502 years, and has absolutely no    \u2502                                 \u2502          \u2502\n\u2502 ideas. Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV. I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 78f28d4e-be0c-4b0e-a1a1-acf08ede9b06
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"78f28d4e-be0c-4b0e-a1a1-acf08ede9b06\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"52f7ab45-2a74-4cc6-adb3-a00d8461741f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"134a7ffffad99a1509ac5fcd3b612b617e0e402600b051835fac7c5b4ede7109\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2f59c366-287b-4ddd-b790-fc8d2d48f737\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bb451522e67acbb4a8fd5b9cdce53312540f6670b598f4e9dc1019ae0557cdd4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9619, \"end_char_idx\": 13647, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV. I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down. She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality. I had no  \u2502                                 \u2502          \u2502\n\u2502 lock on my door; but I had a    \u2502                                 \u2502          \u2502\n\u2502 dresser right next to it, so I  \u2502                                 \u2502          \u2502\n\u2502 pulled one of the drawers out   \u2502                                 \u2502          \u2502\n\u2502 enough so that it caused the    \u2502                                 \u2502          \u2502\n\u2502 door to not be openable. Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2f59c366-287b-4ddd-b790-fc8d2d48f737
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2f59c366-287b-4ddd-b790-fc8d2d48f737\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"78f28d4e-be0c-4b0e-a1a1-acf08ede9b06\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"874fd85ec43080ae03ec5e95f6ae5d99c3a5f84a22e7a99982df52be27b1fe28\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"82940200-efd2-4671-b8d8-2bd152dbcfa0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"52bb2eeeb095ff6270b640f46ac0051aa7251a8fcff9df8632735177dda7925b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11482, \"end_char_idx\": 15475, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality. I had no  \u2502                                 \u2502          \u2502\n\u2502 lock on my door; but I had a    \u2502                                 \u2502          \u2502\n\u2502 dresser right next to it, so I  \u2502                                 \u2502          \u2502\n\u2502 pulled one of the drawers out   \u2502                                 \u2502          \u2502\n\u2502 enough so that it caused the    \u2502                                 \u2502          \u2502\n\u2502 door to not be openable. Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it. But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nMany online implementation details are borrowed from the PPOv2Trainer, which is itself based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031). Here are some additional implementation details:\n\n1. When we turn on the EOS trick (i.e., replacing the score of completions that do not end with an EOS token with a scalar penalty score like `-1`) via `--non_eos_penalty --stop_token eos`, it's possible that the chosen and rejected completions have the same score. In this case, we will naively select the completion with the lower index and the chosen completion.\n\n## Benchmark experiments\n\nTo validate the online DPO implementation works, we ran experiments on the 1B and 6.9B models. Here are the commands we used to run the experiments. We take the SFT / RM models directly from [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 82940200-efd2-4671-b8d8-2bd152dbcfa0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"82940200-efd2-4671-b8d8-2bd152dbcfa0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2f59c366-287b-4ddd-b790-fc8d2d48f737\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"45b66644c084a6bcda048dfadd4c35922bab8f22025890f6ec416af47d911df9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b5294cb6-d831-4768-b10b-1438b19db721\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f82b94ca04fe19f0832cdec05c3f9c9330423661fbbc6827884915d8f822e7c5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13648, \"end_char_idx\": 17303, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nMany online implementation details are borrowed from the PPOv2Trainer, which is itself based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031). Here are some additional implementation details:\n\n1. When we turn on the EOS trick (i.e., replacing the score of completions that do not end with an EOS token with a scalar penalty score like `-1`) via `--non_eos_penalty --stop_token eos`, it's possible that the chosen and rejected completions have the same score. In this case, we will naively select the completion with the lower index and the chosen completion.\n\n## Benchmark experiments\n\nTo validate the online DPO implementation works, we ran experiments on the 1B and 6.9B models. Here are the commands we used to run the experiments. We take the SFT / RM models directly from [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n\n```\n# 1B Online DPO experiment\naccelerate launch --config_file examples/accelerate_configs/deepspeed_zero2.yaml \\\n    examples/scripts/online_dpo.py \\\n    --dataset_name trl-lib/tldr \\\n    --learning_rate 3e-6 \\\n    --output_dir models/minimal/online_dpo_tldr \\\n    --per_device_train_batch_size 16 \\\n    --gradient_accumulation_steps 4 \\\n    --local_rollout_forward_batch_size 32 \\\n    --num_epochs 1 \\\n    --num_mini_batches 1 \\\n    --total_episodes 1000000 \\\n    --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr  \\\n    --sft_model_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr \\\n    --reward_model_path cleanrl/EleutherAI_pythia-1b-deduped__reward__tldr \\\n    --save_strategy no \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n    --beta 0.1 \\\n    --response_length 53 \\\n    --push_to_hub\n\n# 6.9B Online DPO experiment\naccelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml \\\n    examples/scripts/online_dpo.py \\\n    --dataset_name trl-lib/tldr \\\n    --learning_rate 3e-6 \\\n    --output_dir models/minimal/online_dpo_tldr_6.9b \\\n    --per_device_train_batch_size 4 \\\n    --gradient_accumulation_steps 16 \\\n    --local_rollout_forward_batch_size 8 \\\n    --num_epochs 1 \\\n    --num_mini_batches 1 \\\n    --total_episodes 1000000 \\\n    --model_name_or_path EleutherAI/pythia-6.9b-deduped \\\n    --sft_model_path cleanrl/EleutherAI_pythia-6.9b-deduped__sft__tldr \\\n    --reward_model_path cleanrl/EleutherAI_pythia-6.9b-deduped__reward__tldr \\\n    --save_strategy no \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n    --beta 0.1 \\\n    --response_length 53 \\\n    --push_to_hub\n```\n\nCheckpoints and experiment tracking are available at:\n\n- [\ud83e\udd17 Model checkpoint](https://huggingface.co/vwxyzjn/ppo_tldr)\n- [\ud83d\udc1d Tracked experiment](https://wandb.ai/huggingface/trl/runs/dd2o3g35)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b5294cb6-d831-4768-b10b-1438b19db721
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/online_dpo_trainer", "title": "Online DPO Trainer", "tokens": 4521, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b5294cb6-d831-4768-b10b-1438b19db721\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae4d4218-2aa9-413b-bbe4-2665b89b5998\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3a728c5c956951b40498782483a1a6fc9c27dfe14a50ee062e3eb965e3c60518\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"82940200-efd2-4671-b8d8-2bd152dbcfa0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/online_dpo_trainer\", \"title\": \"Online DPO Trainer\", \"tokens\": 4521, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"da1a8778460ebdeb47d563a51c252ee8926d9fb8d35c0f5dd8aa5d05539b1812\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17306, \"end_char_idx\": 19216, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "ref_doc_id": "ae4d4218-2aa9-413b-bbe4-2665b89b5998", "chroma:document": "To evaluate, we use [vLLM](https://github.com/vllm-project/vllm) to load the checkpoints and GPT-4o mini as a judge model to evaluate the generated TL;DR against the reference TL;DR.\nFor more information on how to use judges, see [Judges](judges).\n\n```bash\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 33.00%\npython examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-6.9b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 41.50%\npython examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/online_dpo_tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 62.60%\npython examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/online_dpo_tldr_6.9b --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 74.20%\n```\n\nWe can then plot the RLHF scaling chart.\n\n```python\nimport matplotlib.pyplot as plt\n\ndata = {\n    \"SFT\": [[1e9, 6.9e9], [0.33, 0.415]],\n    \"Online DPO\": [[1e9, 6.9e9], [0.626, 0.742]],\n}\nfor model, (x, y) in data.items():\n    plt.scatter(x, y, label=model)\n\nplt.axhline(y=0.5, color=\"black\", linestyle=\"-.\", label=\"Human reference summary\")\nplt.title(\"RLHF scaling by model size\")\nplt.xlabel(\"Model size\")\nplt.ylabel(\"Win rate against reference summaries\\n(according to GPT-4o mini)\")\nplt.xscale(\"log\")\nplt.xlim(5e8, 1.2e10)\nplt.xticks([1e9, 1e10], [\"1B\", \"10B\"])\nplt.legend()\nplt.grid(True, which=\"both\", ls=\"--\", c=\"0.7\")\nplt.tight_layout()\nplt.savefig(\"plot.png\")\n```\n\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/online_dpo_scaling.png)\n\nThe online DPO checkpoint gets increasingly more win rate as we scale up the model sizes. This is a good sign that the online DPO implementation is working as intended."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 05b3cee7-cdb2-4bdf-b59b-4e156e237ac1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/judges", "title": "Judges", "tokens": 553, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"05b3cee7-cdb2-4bdf-b59b-4e156e237ac1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/judges\", \"title\": \"Judges\", \"tokens\": 553, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"877f15bd-d503-422d-b8df-d7093846a3d2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/judges\", \"title\": \"Judges\", \"tokens\": 553, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3fb9015fa558da6c9cfa52d38e382d6eeda0753f7b98e0b51a1045435480864a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2163, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "877f15bd-d503-422d-b8df-d7093846a3d2", "doc_id": "877f15bd-d503-422d-b8df-d7093846a3d2", "ref_doc_id": "877f15bd-d503-422d-b8df-d7093846a3d2", "chroma:document": "# Judges\n\nTRL provides judges to easily compare two completions.\n\nMake sure to have installed the required dependencies by running:\n\n```bash\npip install trl[llm_judge]\n```\n\n## Using the provided judges\n\nTRL provides several judges out of the box. For example, you can use the `HfPairwiseJudge` to compare two completions using a pre-trained model from the Hugging Face model hub:\n\n```python\nfrom trl import HfPairwiseJudge\n\njudge = HfPairwiseJudge()\njudge.judge(\n    prompts=[\"What is the capital of France?\", \"What is the biggest planet in the solar system?\"],\n    completions=[[\"Paris\", \"Lyon\"], [\"Saturn\", \"Jupiter\"]],\n)  # Outputs: [0, 1]\n```\n\n## Define your own judge\n\nTo define your own judge, we provide several base classes that you can subclass. For rank-based judges, you need to subclass [`BaseRankJudge`] and implement the [`BaseRankJudge.judge`] method. For pairwise judges, you need to subclass [`BasePairJudge`] and implement the [`BasePairJudge.judge`] method. If you want to define a judge that doesn't fit into these categories, you need to subclass [`BaseJudge`] and implement the [`BaseJudge.judge`] method.\n\nAs an example, let's define a pairwise judge that prefers shorter completions:\n\n```python\nfrom trl import BasePairwiseJudge\n\nclass PrefersShorterJudge(BasePairwiseJudge):\n    def judge(self, prompts, completions, shuffle_order=False):\n        return [0 if len(completion[0]) > len(completion[1]) else 1 for completion in completions]\n```\n\nYou can then use this judge as follows:\n\n```python\njudge = PrefersShorterJudge()\njudge.judge(\n    prompts=[\"What is the capital of France?\", \"What is the biggest planet in the solar system?\"],\n    completions=[[\"Paris\", \"The capital of France is Paris.\"], [\"Jupiter is the biggest planet in the solar system.\", \"Jupiter\"]],\n)  # Outputs: [0, 1]\n```\n\n## BaseJudge\n\n[[autodoc]] BaseJudge\n\n## BaseRankJudge\n\n[[autodoc]] BaseRankJudge\n\n## BasePairwiseJudge\n\n[[autodoc]] BasePairwiseJudge\n\n## RandomRankJudge\n\n[[autodoc]] RandomRankJudge\n\n## RandomPairwiseJudge\n\n[[autodoc]] RandomPairwiseJudge\n\n## HfPairwiseJudge\n\n[[autodoc]] HfPairwiseJudge\n\n## OpenAIPairwiseJudge\n\n[[autodoc]] OpenAIPairwiseJudge"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 35192912-b4b4-4841-8d4b-14f40f14b290
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/reward_trainer", "title": "Reward Modeling", "tokens": 889, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"35192912-b4b4-4841-8d4b-14f40f14b290\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/reward_trainer\", \"title\": \"Reward Modeling\", \"tokens\": 889, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"65643d67-d101-4275-ae94-47c6fb11fae3\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/reward_trainer\", \"title\": \"Reward Modeling\", \"tokens\": 889, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"01663f69a87cb0fb41463365e36fd3d044ffae01b4de18a3fb35c8dac45080fa\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3de4f44a-c079-4d1b-9f17-bb5809e97a82\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"999325dbf6239d21eb97d57a1b3c9de76665940263f4ed788f143cc0c4096717\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3031, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "doc_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "ref_doc_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "chroma:document": "# Reward Modeling\n\nTRL supports custom reward modeling for anyone to perform reward modeling on their dataset and model.\n\nCheck out a complete flexible example at [`examples/scripts/reward_modeling.py`](https://github.com/huggingface/trl/tree/main/examples/scripts/reward_modeling.py).\n\n## Expected dataset format\n\nThe [`RewardTrainer`] expects a very specific format for the dataset since the model will be trained on pairs of examples to predict which of the two is preferred. We provide an example from the [`Anthropic/hh-rlhf`](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset below:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/rlhf-antropic-example.png\", width=\"50%\">\n</div>\n\nTherefore the final dataset object should contain two 4 entries at least if you use the default [`RewardDataCollatorWithPadding`] data collator. The entries should be named:\n\n-   `input_ids_chosen`\n-   `attention_mask_chosen`\n-   `input_ids_rejected`\n-   `attention_mask_rejected`\n\n## Using the `RewardTrainer`\n\nAfter preparing your dataset, you can use the [`RewardTrainer`] in the same way as the `Trainer` class from \ud83e\udd17 Transformers.\nYou should pass an `AutoModelForSequenceClassification` model to the [`RewardTrainer`], along with a [`RewardConfig`] which configures the hyperparameters of the training.\n\n### Leveraging \ud83e\udd17 PEFT to train a reward model\n\nJust pass a `peft_config` in the keyword arguments of [`RewardTrainer`], and the trainer should automatically take care of converting the model into a PEFT model!\n\n```python\nfrom peft import LoraConfig, TaskType\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\nfrom trl import RewardTrainer, RewardConfig\n\nmodel = AutoModelForSequenceClassification.from_pretrained(\"gpt2\")\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_CLS,\n    inference_mode=False,\n    r=8,\n    lora_alpha=32,\n    lora_dropout=0.1,\n)\n\n...\n\ntrainer = RewardTrainer(\n    model=model,\n    args=training_args,\n    tokenizer=tokenizer,\n    train_dataset=dataset,\n    peft_config=peft_config,\n)\n\ntrainer.train()\n\n```\n\n### Adding a margin to the loss\n\nAs in the [Llama 2 paper](https://huggingface.co/papers/2307.09288), you can add a margin to the loss by adding a `margin` column to the dataset. The reward collator will automatically pass it through and the loss will be computed accordingly.\n\n```python\ndef add_margin(row):\n    # Assume you have a score_chosen and score_rejected columns that you want to use to compute the margin\n    return {'margin': row['score_chosen'] - row['score_rejected']}\n\ndataset = dataset.map(add_margin)\n```\n\n### Centering rewards\n\nIn many scenarios, it's preferable to ensure that a reward model's output is mean zero. This is often done by first calculating the model's average score and then subtracting it.\n\n[[Eisenstein et al., 2023]](https://huggingface.co/papers/2312.09244) proposed an auxiliary loss function designed to directly learn a centered reward model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3de4f44a-c079-4d1b-9f17-bb5809e97a82
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/reward_trainer", "title": "Reward Modeling", "tokens": 889, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3de4f44a-c079-4d1b-9f17-bb5809e97a82\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/reward_trainer\", \"title\": \"Reward Modeling\", \"tokens\": 889, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"65643d67-d101-4275-ae94-47c6fb11fae3\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/reward_trainer\", \"title\": \"Reward Modeling\", \"tokens\": 889, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"01663f69a87cb0fb41463365e36fd3d044ffae01b4de18a3fb35c8dac45080fa\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"35192912-b4b4-4841-8d4b-14f40f14b290\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/reward_trainer\", \"title\": \"Reward Modeling\", \"tokens\": 889, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"14b0567a8e4f02e88df1abdeba721f6b5457220a7ed48d5fc9c12defa3a460a3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1599, \"end_char_idx\": 3693, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "doc_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "ref_doc_id": "65643d67-d101-4275-ae94-47c6fb11fae3", "chroma:document": "```python\nfrom peft import LoraConfig, TaskType\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer\nfrom trl import RewardTrainer, RewardConfig\n\nmodel = AutoModelForSequenceClassification.from_pretrained(\"gpt2\")\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_CLS,\n    inference_mode=False,\n    r=8,\n    lora_alpha=32,\n    lora_dropout=0.1,\n)\n\n...\n\ntrainer = RewardTrainer(\n    model=model,\n    args=training_args,\n    tokenizer=tokenizer,\n    train_dataset=dataset,\n    peft_config=peft_config,\n)\n\ntrainer.train()\n\n```\n\n### Adding a margin to the loss\n\nAs in the [Llama 2 paper](https://huggingface.co/papers/2307.09288), you can add a margin to the loss by adding a `margin` column to the dataset. The reward collator will automatically pass it through and the loss will be computed accordingly.\n\n```python\ndef add_margin(row):\n    # Assume you have a score_chosen and score_rejected columns that you want to use to compute the margin\n    return {'margin': row['score_chosen'] - row['score_rejected']}\n\ndataset = dataset.map(add_margin)\n```\n\n### Centering rewards\n\nIn many scenarios, it's preferable to ensure that a reward model's output is mean zero. This is often done by first calculating the model's average score and then subtracting it.\n\n[[Eisenstein et al., 2023]](https://huggingface.co/papers/2312.09244) proposed an auxiliary loss function designed to directly learn a centered reward model. This auxiliary loss minimizes the squared sum of the rewards, encouraging the model to naturally produce mean-zero outputs:\n\n$$\\Big( R(p, r_1) + R(p, r_2) \\Big)^2 $$\n\nThis auxiliary loss is combined with the main loss function, weighted by the parameter `center_rewards_coefficient` in the `[RewardConfig]`. By default, this feature is deactivated (`center_rewards_coefficient = None`).\n\n```python\nreward_config = RewardConfig(\n    center_rewards_coefficient=0.01,\n    ...\n)\n```\n\nFor reference results, please refer PR [#1932](https://github.com/huggingface/trl/pull/1932).\n\n## RewardConfig\n\n[[autodoc]] RewardConfig\n\n## RewardTrainer\n\n[[autodoc]] RewardTrainer"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d7eb4ce5-bffb-4996-988d-a4e28049846b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"d7eb4ce5-bffb-4996-988d-a4e28049846b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cad47e91-7c5e-47ac-9a1b-32574008751d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"79b46277667bf2316397a50b2672c75c241041feb3441f5010fc3c48cf2f0fed\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2441, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "# Supervised Fine-tuning Trainer\n\nSupervised fine-tuning (or SFT for short) is a crucial step in RLHF. In TRL we provide an easy-to-use API to create your SFT models and train them with few lines of code on your dataset.\n\nCheck out a complete flexible example at [`examples/scripts/sft.py`](https://github.com/huggingface/trl/tree/main/examples/scripts/sft.py).\nExperimental support for Vision Language Models is also included in the example [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/tree/main/examples/scripts/vsft_llava.py).\n\n## Quickstart\n\nIf you have a dataset hosted on the \ud83e\udd17 Hub, you can easily fine-tune your SFT model using [`SFTTrainer`] from TRL. Let us assume your dataset is `imdb`, the text you want to predict is inside the `text` field of the dataset, and you want to fine-tune the `facebook/opt-350m` model.\nThe following code-snippet takes care of all the data pre-processing and training for you:\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nsft_config = SFTConfig(\n    dataset_text_field=\"text\",\n    max_seq_length=512,\n    output_dir=\"/tmp\",\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\ntrainer.train()\n```\nMake sure to pass the correct value for `max_seq_length` as the default value will be set to `min(tokenizer.model_max_length, 1024)`.\n\nYou can also construct a model outside of the trainer and pass it as follows:\n\n```python\nfrom transformers import AutoModelForCausalLM\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n\nsft_config = SFTConfig(output_dir=\"/tmp\")\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=sft_config,\n)\n\ntrainer.train()\n```\n\nThe above snippets will use the default training arguments from the [`SFTConfig`] class. If you want to modify the defaults pass in your modification to the `SFTConfig` constructor and pass them to the trainer via the `args` argument.\n\n## Advanced usage\n\n### Train on completions only\n\nYou can use the `DataCollatorForCompletionOnlyLM` to train your model on the generated prompts only. Note that this works only in the case when `packing=False`.\nTo instantiate that collator for instruction data, pass a response template and the tokenizer."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: cad47e91-7c5e-47ac-9a1b-32574008751d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"cad47e91-7c5e-47ac-9a1b-32574008751d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d7eb4ce5-bffb-4996-988d-a4e28049846b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"ca831dce5b552e21aa5af5f0e84b3bb679fa749a28d93dad98eb1815a1879cdd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"25173168-bd65-47a4-a6e0-c1305e6211e0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dc51635ad9d1c6d4d3066118f6557e10916e3079b7ba6d27c200d5d7f9a7e9d7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 852, \"end_char_idx\": 3650, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "The following code-snippet takes care of all the data pre-processing and training for you:\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nsft_config = SFTConfig(\n    dataset_text_field=\"text\",\n    max_seq_length=512,\n    output_dir=\"/tmp\",\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\ntrainer.train()\n```\nMake sure to pass the correct value for `max_seq_length` as the default value will be set to `min(tokenizer.model_max_length, 1024)`.\n\nYou can also construct a model outside of the trainer and pass it as follows:\n\n```python\nfrom transformers import AutoModelForCausalLM\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n\nsft_config = SFTConfig(output_dir=\"/tmp\")\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=sft_config,\n)\n\ntrainer.train()\n```\n\nThe above snippets will use the default training arguments from the [`SFTConfig`] class. If you want to modify the defaults pass in your modification to the `SFTConfig` constructor and pass them to the trainer via the `args` argument.\n\n## Advanced usage\n\n### Train on completions only\n\nYou can use the `DataCollatorForCompletionOnlyLM` to train your model on the generated prompts only. Note that this works only in the case when `packing=False`.\nTo instantiate that collator for instruction data, pass a response template and the tokenizer. Here is an example of how it would work to fine-tune `opt-350m` on completions only on the CodeAlpaca dataset:\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer, DataCollatorForCompletionOnlyLM\n\ndataset = load_dataset(\"lucasmccabe-lmi/CodeAlpaca-20k\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\ndef formatting_prompts_func(example):\n    output_texts = []\n    for i in range(len(example['instruction'])):\n        text = f\"### Question: {example['instruction'][i]}\\n ### Answer: {example['output'][i]}\"\n        output_texts.append(text)\n    return output_texts\n\nresponse_template = \" ### Answer:\"\ncollator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=tokenizer)\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=SFTConfig(output_dir=\"/tmp\"),\n    formatting_func=formatting_prompts_func,\n    data_collator=collator,\n)\n\ntrainer.train()\n```\n\nTo instantiate that collator for assistant style conversation data, pass a response template, an instruction template and the tokenizer."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 25173168-bd65-47a4-a6e0-c1305e6211e0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"25173168-bd65-47a4-a6e0-c1305e6211e0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cad47e91-7c5e-47ac-9a1b-32574008751d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"0e10b669c65006f0547c2ed69c02b1a020e0e1e580b54ca5be9a74f10517f089\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b265c072-a6f9-408a-a1a7-9a1a4c5d4346\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bc56e892b1e7aef3891ddb822de27d82909e497323e2d97ab45e1657e65c9677\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1989, \"end_char_idx\": 4968, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "If you want to modify the defaults pass in your modification to the `SFTConfig` constructor and pass them to the trainer via the `args` argument.\n\n## Advanced usage\n\n### Train on completions only\n\nYou can use the `DataCollatorForCompletionOnlyLM` to train your model on the generated prompts only. Note that this works only in the case when `packing=False`.\nTo instantiate that collator for instruction data, pass a response template and the tokenizer. Here is an example of how it would work to fine-tune `opt-350m` on completions only on the CodeAlpaca dataset:\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer, DataCollatorForCompletionOnlyLM\n\ndataset = load_dataset(\"lucasmccabe-lmi/CodeAlpaca-20k\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\ndef formatting_prompts_func(example):\n    output_texts = []\n    for i in range(len(example['instruction'])):\n        text = f\"### Question: {example['instruction'][i]}\\n ### Answer: {example['output'][i]}\"\n        output_texts.append(text)\n    return output_texts\n\nresponse_template = \" ### Answer:\"\ncollator = DataCollatorForCompletionOnlyLM(response_template, tokenizer=tokenizer)\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=SFTConfig(output_dir=\"/tmp\"),\n    formatting_func=formatting_prompts_func,\n    data_collator=collator,\n)\n\ntrainer.train()\n```\n\nTo instantiate that collator for assistant style conversation data, pass a response template, an instruction template and the tokenizer. Here is an example of how it would work to fine-tune `opt-350m` on assistant completions only on the Open Assistant Guanaco dataset:\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer, DataCollatorForCompletionOnlyLM\n\ndataset = load_dataset(\"timdettmers/openassistant-guanaco\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\ninstruction_template = \"### Human:\"\nresponse_template = \"### Assistant:\"\ncollator = DataCollatorForCompletionOnlyLM(instruction_template=instruction_template, response_template=response_template, tokenizer=tokenizer, mlm=False)\n\ntrainer = SFTTrainer(\n    model,\n    args=SFTConfig(\n        output_dir=\"/tmp\",\n        dataset_text_field = \"text\",\n    ),\n    train_dataset=dataset,\n    data_collator=collator,\n)\n\ntrainer.train()\n```\n\nMake sure to have a `pad_token_id` which is different from `eos_token_id` which can result in the model not properly predicting EOS (End of Sentence) tokens during generation.\n\n#### Using token_ids directly for `response_template`\n\nSome tokenizers like Llama 2 (`meta-llama/Llama-2-XXb-hf`) tokenize sequences differently depending on whether they have context or not."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b265c072-a6f9-408a-a1a7-9a1a4c5d4346
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"b265c072-a6f9-408a-a1a7-9a1a4c5d4346\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"25173168-bd65-47a4-a6e0-c1305e6211e0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"3e3566d706bd8d16afe292affae1a96e056f7bc181c5f1b13755519180b93c6e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e3d4d940-61da-430d-a6ec-26640473a8d8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"526dc27809ca8302a563981c06eaa0ebd93f8a10fe77395e0fde13a8ed0adf0b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3651, \"end_char_idx\": 6566, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Here is an example of how it would work to fine-tune `opt-350m` on assistant completions only on the Open Assistant Guanaco dataset:\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer, DataCollatorForCompletionOnlyLM\n\ndataset = load_dataset(\"timdettmers/openassistant-guanaco\", split=\"train\")\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\ninstruction_template = \"### Human:\"\nresponse_template = \"### Assistant:\"\ncollator = DataCollatorForCompletionOnlyLM(instruction_template=instruction_template, response_template=response_template, tokenizer=tokenizer, mlm=False)\n\ntrainer = SFTTrainer(\n    model,\n    args=SFTConfig(\n        output_dir=\"/tmp\",\n        dataset_text_field = \"text\",\n    ),\n    train_dataset=dataset,\n    data_collator=collator,\n)\n\ntrainer.train()\n```\n\nMake sure to have a `pad_token_id` which is different from `eos_token_id` which can result in the model not properly predicting EOS (End of Sentence) tokens during generation.\n\n#### Using token_ids directly for `response_template`\n\nSome tokenizers like Llama 2 (`meta-llama/Llama-2-XXb-hf`) tokenize sequences differently depending on whether they have context or not. For example:\n\n```python\nfrom transformers import AutoTokenizer\ntokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-hf\")\n\ndef print_tokens_with_ids(txt):\n    tokens = tokenizer.tokenize(txt, add_special_tokens=False)\n    token_ids = tokenizer.encode(txt, add_special_tokens=False)\n    print(list(zip(tokens, token_ids)))\n\nprompt = \"\"\"### User: Hello\\n\\n### Assistant: Hi, how can I help you?\"\"\"\nprint_tokens_with_ids(prompt)  # [..., ('\u2581Hello', 15043), ('<0x0A>', 13), ('<0x0A>', 13), ('##', 2277), ('#', 29937), ('\u2581Ass', 4007), ('istant', 22137), (':', 29901), ...]\n\nresponse_template = \"### Assistant:\"\nprint_tokens_with_ids(response_template)  # [('\u2581###', 835), ('\u2581Ass', 4007), ('istant', 22137), (':', 29901)]\n```\n\nIn this case, and due to lack of context in `response_template`, the same string (\"### Assistant:\") is tokenized differently:\n\n    - Text (with context): `[2277, 29937, 4007, 22137, 29901]`\n    - `response_template` (without context): `[835, 4007, 22137, 29901]`\n\nThis will lead to an error when the `DataCollatorForCompletionOnlyLM` does not find the `response_template` in the dataset example text:\n\n```\nRuntimeError: Could not find response key [835, 4007, 22137, 29901] in token IDs tensor([    1,   835,  ...])\n```\n\n\nTo solve this, you can tokenize the `response_template` with the same context as in the dataset, truncate it as needed and pass the `token_ids` directly to the `response_template` argument of the `DataCollatorForCompletionOnlyLM` class. For example:\n\n```python\nresponse_template_with_context = \"\\n### Assistant:\"  # We added context here: \"\\n\"."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: e3d4d940-61da-430d-a6ec-26640473a8d8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"e3d4d940-61da-430d-a6ec-26640473a8d8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b265c072-a6f9-408a-a1a7-9a1a4c5d4346\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"b4c14841ebf0b2a978f268576ff9e438116afed905e54c7cfa0e3ea31a855f8e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a34fef11-5909-4d13-b4db-179178f1ba4e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5b62e8489586b3e69cfb6624456fe2e321ab6da3cf0bd18086ba87e10c78f4ce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5376, \"end_char_idx\": 7926, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "print_tokens_with_ids(prompt)  # [..., ('\u2581Hello', 15043), ('<0x0A>', 13), ('<0x0A>', 13), ('##', 2277), ('#', 29937), ('\u2581Ass', 4007), ('istant', 22137), (':', 29901), ...]\n\nresponse_template = \"### Assistant:\"\nprint_tokens_with_ids(response_template)  # [('\u2581###', 835), ('\u2581Ass', 4007), ('istant', 22137), (':', 29901)]\n```\n\nIn this case, and due to lack of context in `response_template`, the same string (\"### Assistant:\") is tokenized differently:\n\n    - Text (with context): `[2277, 29937, 4007, 22137, 29901]`\n    - `response_template` (without context): `[835, 4007, 22137, 29901]`\n\nThis will lead to an error when the `DataCollatorForCompletionOnlyLM` does not find the `response_template` in the dataset example text:\n\n```\nRuntimeError: Could not find response key [835, 4007, 22137, 29901] in token IDs tensor([    1,   835,  ...])\n```\n\n\nTo solve this, you can tokenize the `response_template` with the same context as in the dataset, truncate it as needed and pass the `token_ids` directly to the `response_template` argument of the `DataCollatorForCompletionOnlyLM` class. For example:\n\n```python\nresponse_template_with_context = \"\\n### Assistant:\"  # We added context here: \"\\n\". This is enough for this tokenizer\nresponse_template_ids = tokenizer.encode(response_template_with_context, add_special_tokens=False)[2:]  # Now we have it like in the dataset texts: `[2277, 29937, 4007, 22137, 29901]`\n\ndata_collator = DataCollatorForCompletionOnlyLM(response_template_ids, tokenizer=tokenizer)\n```\n\n### Add Special Tokens for Chat Format\n\nAdding special tokens to a language model is crucial for training chat models. These tokens are added between the different roles in a conversation, such as the user, assistant, and system and help the model recognize the structure and flow of a conversation. This setup is essential for enabling the model to generate coherent and contextually appropriate responses in a chat environment. \nThe [`setup_chat_format`] function in `trl` easily sets up a model and tokenizer for conversational AI tasks. This function:\n- Adds special tokens to the tokenizer, e.g. `<|im_start|>` and `<|im_end|>`, to indicate the start and end of a conversation.\n- Resizes the model\u2019s embedding layer to accommodate the new tokens.\n- Sets the `chat_template` of the tokenizer, which is used to format the input data into a chat-like format. The default is `chatml` from OpenAI.\n- _optionally_ you can pass `resize_to_multiple_of` to resize the embedding layer to a multiple of the `resize_to_multiple_of` argument, e.g. 64."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a34fef11-5909-4d13-b4db-179178f1ba4e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"a34fef11-5909-4d13-b4db-179178f1ba4e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e3d4d940-61da-430d-a6ec-26640473a8d8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"36e581c94aee832d177df0740898c060ff12b73a36c0319f4c892cb3bbaeaf51\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"be18e574-9e0e-4ba1-8320-60eceff453f2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dc0fe6a248f0c9c63e13559c887c3253e96ab13574972c1885bb13e50ec11362\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6222, \"end_char_idx\": 9309, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "To solve this, you can tokenize the `response_template` with the same context as in the dataset, truncate it as needed and pass the `token_ids` directly to the `response_template` argument of the `DataCollatorForCompletionOnlyLM` class. For example:\n\n```python\nresponse_template_with_context = \"\\n### Assistant:\"  # We added context here: \"\\n\". This is enough for this tokenizer\nresponse_template_ids = tokenizer.encode(response_template_with_context, add_special_tokens=False)[2:]  # Now we have it like in the dataset texts: `[2277, 29937, 4007, 22137, 29901]`\n\ndata_collator = DataCollatorForCompletionOnlyLM(response_template_ids, tokenizer=tokenizer)\n```\n\n### Add Special Tokens for Chat Format\n\nAdding special tokens to a language model is crucial for training chat models. These tokens are added between the different roles in a conversation, such as the user, assistant, and system and help the model recognize the structure and flow of a conversation. This setup is essential for enabling the model to generate coherent and contextually appropriate responses in a chat environment. \nThe [`setup_chat_format`] function in `trl` easily sets up a model and tokenizer for conversational AI tasks. This function:\n- Adds special tokens to the tokenizer, e.g. `<|im_start|>` and `<|im_end|>`, to indicate the start and end of a conversation.\n- Resizes the model\u2019s embedding layer to accommodate the new tokens.\n- Sets the `chat_template` of the tokenizer, which is used to format the input data into a chat-like format. The default is `chatml` from OpenAI.\n- _optionally_ you can pass `resize_to_multiple_of` to resize the embedding layer to a multiple of the `resize_to_multiple_of` argument, e.g. 64. If you want to see more formats being supported in the future, please open a GitHub issue on [trl](https://github.com/huggingface/trl)\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom trl import setup_chat_format\n\n# Load model and tokenizer\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\n# Set up the chat format with default 'chatml' format\nmodel, tokenizer = setup_chat_format(model, tokenizer)\n\n```\n\nWith our model and tokenizer set up, we can now fine-tune our model on a conversational dataset. Below is an example of how a dataset can be formatted for fine-tuning. \n\n### Dataset format support\n\nThe [`SFTTrainer`] supports popular dataset formats. This allows you to pass the dataset to the trainer without any pre-processing directly. The following formats are supported:\n* conversational format\n```json\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"What's the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"Who wrote 'Romeo and Juliet'?\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"How far is the Moon from Earth?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: be18e574-9e0e-4ba1-8320-60eceff453f2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"be18e574-9e0e-4ba1-8320-60eceff453f2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a34fef11-5909-4d13-b4db-179178f1ba4e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"0597ca2a4579ec111a3a84590fc53a550b0b284bf60a6d9b0235703bebe0406a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"bf9b80d9-e260-4700-8734-1f33c0ffa25d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"68bb74f56bb51aa576e8cb559e5e2fa76437c53cbbf41322fd9f8cb5aab6b3fc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7744, \"end_char_idx\": 10759, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "The default is `chatml` from OpenAI.\n- _optionally_ you can pass `resize_to_multiple_of` to resize the embedding layer to a multiple of the `resize_to_multiple_of` argument, e.g. 64. If you want to see more formats being supported in the future, please open a GitHub issue on [trl](https://github.com/huggingface/trl)\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nfrom trl import setup_chat_format\n\n# Load model and tokenizer\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\n# Set up the chat format with default 'chatml' format\nmodel, tokenizer = setup_chat_format(model, tokenizer)\n\n```\n\nWith our model and tokenizer set up, we can now fine-tune our model on a conversational dataset. Below is an example of how a dataset can be formatted for fine-tuning. \n\n### Dataset format support\n\nThe [`SFTTrainer`] supports popular dataset formats. This allows you to pass the dataset to the trainer without any pre-processing directly. The following formats are supported:\n* conversational format\n```json\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"What's the capital of France?\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"Who wrote 'Romeo and Juliet'?\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n{\"messages\": [{\"role\": \"system\", \"content\": \"You are helpful\"}, {\"role\": \"user\", \"content\": \"How far is the Moon from Earth?\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n```\n* instruction format\n```json\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n```\n\nIf your dataset uses one of the above formats, you can directly pass it to the trainer without pre-processing. The [`SFTTrainer`] will then format the dataset for you using the defined format from the model's tokenizer with the [apply_chat_template](https://huggingface.co/docs/transformers/main/en/chat_templating#templates-for-chat-models) method. \n\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\n...\n\n# load jsonl dataset\ndataset = load_dataset(\"json\", data_files=\"path/to/dataset.jsonl\", split=\"train\")\n# load dataset from the HuggingFace Hub\ndataset = load_dataset(\"philschmid/dolly-15k-oai-style\", split=\"train\")\n\n...\n\nsft_config = SFTConfig(packing=True)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    args=sft_config,\n    train_dataset=dataset,\n)\n```\n\nIf the dataset is not in one of those format you can either preprocess the dataset to match the formatting or pass a formatting function to the SFTTrainer to do it for you. Let's have a look.\n\n\n### Format your input prompts\n\nFor instruction fine-tuning, it is quite common to have two columns inside the dataset: one for the prompt & the other for the response."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: bf9b80d9-e260-4700-8734-1f33c0ffa25d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"bf9b80d9-e260-4700-8734-1f33c0ffa25d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"be18e574-9e0e-4ba1-8320-60eceff453f2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"93ca685596cba11268138238c5b8d523d4c36cae62445c8dc0fe7fbbf87ebfde\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8153222b-1942-4a7d-8bb5-f08d0d69cd44\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"65ec6533565b2728a122b8502d1acef169ffadb3ae941d6f289322923b59e1c8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9309, \"end_char_idx\": 12193, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "\"}, {\"role\": \"assistant\", \"content\": \"...\"}]}\n```\n* instruction format\n```json\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n{\"prompt\": \"<prompt text>\", \"completion\": \"<ideal generated text>\"}\n```\n\nIf your dataset uses one of the above formats, you can directly pass it to the trainer without pre-processing. The [`SFTTrainer`] will then format the dataset for you using the defined format from the model's tokenizer with the [apply_chat_template](https://huggingface.co/docs/transformers/main/en/chat_templating#templates-for-chat-models) method. \n\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\n...\n\n# load jsonl dataset\ndataset = load_dataset(\"json\", data_files=\"path/to/dataset.jsonl\", split=\"train\")\n# load dataset from the HuggingFace Hub\ndataset = load_dataset(\"philschmid/dolly-15k-oai-style\", split=\"train\")\n\n...\n\nsft_config = SFTConfig(packing=True)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    args=sft_config,\n    train_dataset=dataset,\n)\n```\n\nIf the dataset is not in one of those format you can either preprocess the dataset to match the formatting or pass a formatting function to the SFTTrainer to do it for you. Let's have a look.\n\n\n### Format your input prompts\n\nFor instruction fine-tuning, it is quite common to have two columns inside the dataset: one for the prompt & the other for the response.\nThis allows people to format examples like [Stanford-Alpaca](https://github.com/tatsu-lab/stanford_alpaca) did as follows:\n```bash\nBelow is an instruction ...\n\n### Instruction\n{prompt}\n\n### Response:\n{completion}\n```\nLet us assume your dataset has two fields, `question` and `answer`. Therefore you can just run:\n```python\n...\ndef formatting_prompts_func(example):\n    output_texts = []\n    for i in range(len(example['question'])):\n        text = f\"### Question: {example['question'][i]}\\n ### Answer: {example['answer'][i]}\"\n        output_texts.append(text)\n    return output_texts\n\ntrainer = SFTTrainer(\n    model,\n    args=sft_config,\n    train_dataset=dataset,\n    formatting_func=formatting_prompts_func,\n)\n\ntrainer.train()\n```\nTo properly format your input make sure to process all the examples by looping over them and returning a list of processed text. Check out a full example of how to use SFTTrainer on alpaca dataset [here](https://github.com/huggingface/trl/pull/444#issue-1760952763)\n\n### Packing dataset ([`ConstantLengthDataset`])\n\n[`SFTTrainer`] supports _example packing_, where multiple short examples are packed in the same input sequence to increase training efficiency. This is done with the [`ConstantLengthDataset`] utility class that returns constant length chunks of tokens from a stream of examples. To enable the usage of this dataset class, simply pass `packing=True` to the [`SFTConfig`] constructor."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 8153222b-1942-4a7d-8bb5-f08d0d69cd44
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"8153222b-1942-4a7d-8bb5-f08d0d69cd44\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"bf9b80d9-e260-4700-8734-1f33c0ffa25d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"20f51a81c054133f3a6658f68c7bd5647d17f3a93d872db164a3f368ae9be262\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a4361bf0-2970-4170-a615-654fcfe16179\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4cc91d31df254813f1462002ea0b423f22044784eaf173210fbe0295db0d6f2d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10592, \"end_char_idx\": 13757, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "### Format your input prompts\n\nFor instruction fine-tuning, it is quite common to have two columns inside the dataset: one for the prompt & the other for the response.\nThis allows people to format examples like [Stanford-Alpaca](https://github.com/tatsu-lab/stanford_alpaca) did as follows:\n```bash\nBelow is an instruction ...\n\n### Instruction\n{prompt}\n\n### Response:\n{completion}\n```\nLet us assume your dataset has two fields, `question` and `answer`. Therefore you can just run:\n```python\n...\ndef formatting_prompts_func(example):\n    output_texts = []\n    for i in range(len(example['question'])):\n        text = f\"### Question: {example['question'][i]}\\n ### Answer: {example['answer'][i]}\"\n        output_texts.append(text)\n    return output_texts\n\ntrainer = SFTTrainer(\n    model,\n    args=sft_config,\n    train_dataset=dataset,\n    formatting_func=formatting_prompts_func,\n)\n\ntrainer.train()\n```\nTo properly format your input make sure to process all the examples by looping over them and returning a list of processed text. Check out a full example of how to use SFTTrainer on alpaca dataset [here](https://github.com/huggingface/trl/pull/444#issue-1760952763)\n\n### Packing dataset ([`ConstantLengthDataset`])\n\n[`SFTTrainer`] supports _example packing_, where multiple short examples are packed in the same input sequence to increase training efficiency. This is done with the [`ConstantLengthDataset`] utility class that returns constant length chunks of tokens from a stream of examples. To enable the usage of this dataset class, simply pass `packing=True` to the [`SFTConfig`] constructor.\n\n```python\n...\nsft_config = SFTConfig(packing=True, dataset_text_field=\"text\",)\n\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config\n)\n\ntrainer.train()\n```\n\nNote that if you use a packed dataset and if you pass `max_steps` in the training arguments you will probably train your models for more than few epochs, depending on the way you have configured the packed dataset and the training protocol. Double check that you know and understand what you are doing.\nIf you don't want to pack your `eval_dataset`, you can pass `eval_packing=False` to the `SFTConfig` init method.\n\n#### Customize your prompts using packed dataset\n\nIf your dataset has several fields that you want to combine, for example if the dataset has `question` and `answer` fields and you want to combine them, you can pass a formatting function to the trainer that will take care of that. For example:\n\n```python\ndef formatting_func(example):\n    text = f\"### Question: {example['question']}\\n ### Answer: {example['answer']}\"\n    return text\n\nsft_config = SFTConfig(packing=True)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n    formatting_func=formatting_func\n)\n\ntrainer.train()\n```\nYou can also customize the [`ConstantLengthDataset`] much more by directly passing the arguments to the [`SFTConfig`] constructor. Please refer to that class' signature for more information.\n\n### Control over the pretrained model\n\nYou can directly pass the kwargs of the `from_pretrained()` method to the [`SFTConfig`]."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a4361bf0-2970-4170-a615-654fcfe16179
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"a4361bf0-2970-4170-a615-654fcfe16179\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8153222b-1942-4a7d-8bb5-f08d0d69cd44\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"ca6421f342e7c70e3ae72a636d9566def8883773a8500c5dd0a9ccf1209607c1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"acdf97cb-d534-4963-83f3-a94d7b8fef85\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"188031603ede66d34392a42fb135abd79d4877937972f817a147c230a9197c5d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12090, \"end_char_idx\": 15243, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "To enable the usage of this dataset class, simply pass `packing=True` to the [`SFTConfig`] constructor.\n\n```python\n...\nsft_config = SFTConfig(packing=True, dataset_text_field=\"text\",)\n\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config\n)\n\ntrainer.train()\n```\n\nNote that if you use a packed dataset and if you pass `max_steps` in the training arguments you will probably train your models for more than few epochs, depending on the way you have configured the packed dataset and the training protocol. Double check that you know and understand what you are doing.\nIf you don't want to pack your `eval_dataset`, you can pass `eval_packing=False` to the `SFTConfig` init method.\n\n#### Customize your prompts using packed dataset\n\nIf your dataset has several fields that you want to combine, for example if the dataset has `question` and `answer` fields and you want to combine them, you can pass a formatting function to the trainer that will take care of that. For example:\n\n```python\ndef formatting_func(example):\n    text = f\"### Question: {example['question']}\\n ### Answer: {example['answer']}\"\n    return text\n\nsft_config = SFTConfig(packing=True)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n    formatting_func=formatting_func\n)\n\ntrainer.train()\n```\nYou can also customize the [`ConstantLengthDataset`] much more by directly passing the arguments to the [`SFTConfig`] constructor. Please refer to that class' signature for more information.\n\n### Control over the pretrained model\n\nYou can directly pass the kwargs of the `from_pretrained()` method to the [`SFTConfig`]. For example, if you want to load a model in a different precision, analogous to\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.bfloat16)\n\n...\n\nsft_config = SFTConfig(\n    model_init_kwargs={\n        \"torch_dtype\": \"bfloat16\",\n    },\n    output_dir=\"/tmp\",\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\n\ntrainer.train()\n```\nNote that all keyword arguments of `from_pretrained()` are supported.\n\n### Training adapters\n\nWe also support tight integration with \ud83e\udd17 PEFT library so that any user can conveniently train adapters and share them on the Hub instead of training the entire model\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\nfrom peft import LoraConfig\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\npeft_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\ntrainer = SFTTrainer(\n    \"EleutherAI/gpt-neo-125m\",\n    train_dataset=dataset,\n    args=SFTConfig(output_dir=\"/tmp\"),\n    peft_config=peft_config\n)\n\ntrainer.train()\n```\n\nYou can also continue training your `PeftModel`. For that, first load a `PeftModel` outside `SFTTrainer` and pass it directly to the trainer without the `peft_config` argument being passed.\n\n### Training adapters with base 8 bit models\n\nFor that, you need to first load your 8 bit model outside the Trainer and pass a `PeftConfig` to the trainer."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: acdf97cb-d534-4963-83f3-a94d7b8fef85
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"acdf97cb-d534-4963-83f3-a94d7b8fef85\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a4361bf0-2970-4170-a615-654fcfe16179\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"26a18d251213aae288fcfdc7bf9a78954673b2284973ecb12bfdfca5e3bd88b9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2d4be494-43d7-4b7a-802a-7423e5a47a6b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"baa0b8ad473b1b34dfde3b9e1112276256e77efec49a6c6fdd03065991c5b353\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13758, \"end_char_idx\": 16374, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "For example, if you want to load a model in a different precision, analogous to\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.bfloat16)\n\n...\n\nsft_config = SFTConfig(\n    model_init_kwargs={\n        \"torch_dtype\": \"bfloat16\",\n    },\n    output_dir=\"/tmp\",\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\n\ntrainer.train()\n```\nNote that all keyword arguments of `from_pretrained()` are supported.\n\n### Training adapters\n\nWe also support tight integration with \ud83e\udd17 PEFT library so that any user can conveniently train adapters and share them on the Hub instead of training the entire model\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\nfrom peft import LoraConfig\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\npeft_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\ntrainer = SFTTrainer(\n    \"EleutherAI/gpt-neo-125m\",\n    train_dataset=dataset,\n    args=SFTConfig(output_dir=\"/tmp\"),\n    peft_config=peft_config\n)\n\ntrainer.train()\n```\n\nYou can also continue training your `PeftModel`. For that, first load a `PeftModel` outside `SFTTrainer` and pass it directly to the trainer without the `peft_config` argument being passed.\n\n### Training adapters with base 8 bit models\n\nFor that, you need to first load your 8 bit model outside the Trainer and pass a `PeftConfig` to the trainer. For example:\n\n```python\n...\n\npeft_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    \"EleutherAI/gpt-neo-125m\",\n    load_in_8bit=True,\n    device_map=\"auto\",\n)\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=SFTConfig(),\n    peft_config=peft_config,\n)\n\ntrainer.train()\n```\n\n## Using Flash Attention and Flash Attention 2\n\nYou can benefit from Flash Attention 1 & 2 using SFTTrainer out of the box with minimal changes of code.\nFirst, to make sure you have all the latest features from transformers, install transformers from source\n\n```bash\npip install -U git+https://github.com/huggingface/transformers.git\n```\n\nNote that Flash Attention only works on GPU now and under half-precision regime (when using adapters, base model loaded in half-precision)\nNote also both features are perfectly compatible with other tools such as quantization.\n\n### Using Flash-Attention 1\n\nFor Flash Attention 1 you can use the `BetterTransformer` API and force-dispatch the API to use Flash Attention kernel."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2d4be494-43d7-4b7a-802a-7423e5a47a6b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"2d4be494-43d7-4b7a-802a-7423e5a47a6b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"acdf97cb-d534-4963-83f3-a94d7b8fef85\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"751bdf12628edb2fc9b19c281d5dfddfb6fc7f42e0abfd0270f87716b7cb1636\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6652b3ff-fedf-439f-9516-fad4140e517a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d7c9084241c5eb34060a9b9441f986f61de2e72daaa6b886b23e20bdf7d443b8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14946, \"end_char_idx\": 17290, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "For that, first load a `PeftModel` outside `SFTTrainer` and pass it directly to the trainer without the `peft_config` argument being passed.\n\n### Training adapters with base 8 bit models\n\nFor that, you need to first load your 8 bit model outside the Trainer and pass a `PeftConfig` to the trainer. For example:\n\n```python\n...\n\npeft_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    \"EleutherAI/gpt-neo-125m\",\n    load_in_8bit=True,\n    device_map=\"auto\",\n)\n\ntrainer = SFTTrainer(\n    model,\n    train_dataset=dataset,\n    args=SFTConfig(),\n    peft_config=peft_config,\n)\n\ntrainer.train()\n```\n\n## Using Flash Attention and Flash Attention 2\n\nYou can benefit from Flash Attention 1 & 2 using SFTTrainer out of the box with minimal changes of code.\nFirst, to make sure you have all the latest features from transformers, install transformers from source\n\n```bash\npip install -U git+https://github.com/huggingface/transformers.git\n```\n\nNote that Flash Attention only works on GPU now and under half-precision regime (when using adapters, base model loaded in half-precision)\nNote also both features are perfectly compatible with other tools such as quantization.\n\n### Using Flash-Attention 1\n\nFor Flash Attention 1 you can use the `BetterTransformer` API and force-dispatch the API to use Flash Attention kernel. First, install the latest optimum package:\n\n```bash\npip install -U optimum\n```\n\nOnce you have loaded your model, wrap the `trainer.train()` call under the `with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):` context manager:\n\n```diff\n...\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    trainer.train()\n```\n\nNote that you cannot train your model using Flash Attention 1 on an arbitrary dataset as `torch.scaled_dot_product_attention` does not support training with padding tokens if you use Flash Attention kernels. Therefore you can only use that feature with `packing=True`. If your dataset contains padding tokens, consider switching to Flash Attention 2 integration.\n\nBelow are some numbers you can get in terms of speedup and memory efficiency, using Flash Attention 1, on a single NVIDIA-T4 16GB."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 6652b3ff-fedf-439f-9516-fad4140e517a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"6652b3ff-fedf-439f-9516-fad4140e517a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2d4be494-43d7-4b7a-802a-7423e5a47a6b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"e378e35e5e253a85532b36850becb3e3b100d3b5688aec92cb69861f62b84cfd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8b333075-1db9-4b89-9a78-63b7832ecd97\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"01a0f74a3d8a4fc7b7fcfddfd443b68248345c4a1c56bbe9ddc8d551c4c24c99\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15812, \"end_char_idx\": 18724, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "First, to make sure you have all the latest features from transformers, install transformers from source\n\n```bash\npip install -U git+https://github.com/huggingface/transformers.git\n```\n\nNote that Flash Attention only works on GPU now and under half-precision regime (when using adapters, base model loaded in half-precision)\nNote also both features are perfectly compatible with other tools such as quantization.\n\n### Using Flash-Attention 1\n\nFor Flash Attention 1 you can use the `BetterTransformer` API and force-dispatch the API to use Flash Attention kernel. First, install the latest optimum package:\n\n```bash\npip install -U optimum\n```\n\nOnce you have loaded your model, wrap the `trainer.train()` call under the `with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):` context manager:\n\n```diff\n...\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    trainer.train()\n```\n\nNote that you cannot train your model using Flash Attention 1 on an arbitrary dataset as `torch.scaled_dot_product_attention` does not support training with padding tokens if you use Flash Attention kernels. Therefore you can only use that feature with `packing=True`. If your dataset contains padding tokens, consider switching to Flash Attention 2 integration.\n\nBelow are some numbers you can get in terms of speedup and memory efficiency, using Flash Attention 1, on a single NVIDIA-T4 16GB.\n\n| use_flash_attn_1 | model_name        | max_seq_len | batch_size | time per training step |\n| ---------------- | ----------------- | ----------- | ---------- | ---------------------- |\n| x                | facebook/opt-350m | 2048        | 8          | ~59.1s                 |\n|                  | facebook/opt-350m | 2048        | 8          | **OOM**                |\n| x                | facebook/opt-350m | 2048        | 4          | ~30.3s                 |\n|                  | facebook/opt-350m | 2048        | 4          | ~148.9s                |\n\n### Using Flash Attention-2\n\nTo use Flash Attention 2, first install the latest `flash-attn` package:\n\n```bash\npip install -U flash-attn\n```\n\nAnd add `attn_implementation=\"flash_attention_2\"` when calling `from_pretrained`:\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\"\n)\n```\n\nIf you don't use quantization, make sure your model is loaded in half-precision and dispatch your model on a supported GPU device.\nAfter loading your model, you can either train it as it is, or attach adapters and train adapters on it in case your model is quantized.\n\nIn contrast to Flash Attention 1, the integration makes it possible to train your model on an arbitrary dataset that also includes padding tokens.\n\n\n### Using model creation utility\n\nWe included a utility function to create your model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 8b333075-1db9-4b89-9a78-63b7832ecd97
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"8b333075-1db9-4b89-9a78-63b7832ecd97\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6652b3ff-fedf-439f-9516-fad4140e517a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"9a1ec360d9c050b8fabbc6b7762acedc2e09c021042abb341a427ce56d4886fd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dbe0b07c-6ab9-4a73-a81d-2e38d094c7b1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"50ad2bb8ed43e6530409ebd84c10bb5968b1c8703835350ad990ce89d6b9f2c9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17004, \"end_char_idx\": 20162, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Therefore you can only use that feature with `packing=True`. If your dataset contains padding tokens, consider switching to Flash Attention 2 integration.\n\nBelow are some numbers you can get in terms of speedup and memory efficiency, using Flash Attention 1, on a single NVIDIA-T4 16GB.\n\n| use_flash_attn_1 | model_name        | max_seq_len | batch_size | time per training step |\n| ---------------- | ----------------- | ----------- | ---------- | ---------------------- |\n| x                | facebook/opt-350m | 2048        | 8          | ~59.1s                 |\n|                  | facebook/opt-350m | 2048        | 8          | **OOM**                |\n| x                | facebook/opt-350m | 2048        | 4          | ~30.3s                 |\n|                  | facebook/opt-350m | 2048        | 4          | ~148.9s                |\n\n### Using Flash Attention-2\n\nTo use Flash Attention 2, first install the latest `flash-attn` package:\n\n```bash\npip install -U flash-attn\n```\n\nAnd add `attn_implementation=\"flash_attention_2\"` when calling `from_pretrained`:\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\"\n)\n```\n\nIf you don't use quantization, make sure your model is loaded in half-precision and dispatch your model on a supported GPU device.\nAfter loading your model, you can either train it as it is, or attach adapters and train adapters on it in case your model is quantized.\n\nIn contrast to Flash Attention 1, the integration makes it possible to train your model on an arbitrary dataset that also includes padding tokens.\n\n\n### Using model creation utility\n\nWe included a utility function to create your model.\n\n[[autodoc]] ModelConfig\n\n```python\nfrom trl import ModelConfig, SFTTrainer, get_kbit_device_map, get_peft_config, get_quantization_config\nmodel_config = ModelConfig(\n    model_name_or_path=\"facebook/opt-350m\"\n    attn_implementation=None, # or \"flash_attention_2\"\n)\ntorch_dtype = (\n    model_config.torch_dtype\n    if model_config.torch_dtype in [\"auto\", None]\n    else getattr(torch, model_config.torch_dtype)\n)\nquantization_config = get_quantization_config(model_config)\nmodel_kwargs = dict(\n    revision=model_config.model_revision,\n    trust_remote_code=model_config.trust_remote_code,\n    attn_implementation=model_config.attn_implementation,\n    torch_dtype=torch_dtype,\n    use_cache=False if training_args.gradient_checkpointing else True,\n    device_map=get_kbit_device_map() if quantization_config is not None else None,\n    quantization_config=quantization_config,\n)\nmodel = AutoModelForCausalLM.from_pretrained(model_config.model_name_or_path, **model_kwargs)\ntrainer = SFTTrainer(\n    ...,\n    model=model_config.model_name_or_path,\n    peft_config=get_peft_config(model_config),\n)\n```\n\n### Enhance the model's performances using NEFTune\n\nNEFTune is a technique to boost the performance of chat models and was introduced by the paper [\"NEFTune: Noisy Embeddings Improve Instruction Finetuning\"](https://huggingface.co/papers/2310.05914) from Jain et al. it consists of adding noise to the embedding vectors during training."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: dbe0b07c-6ab9-4a73-a81d-2e38d094c7b1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"dbe0b07c-6ab9-4a73-a81d-2e38d094c7b1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8b333075-1db9-4b89-9a78-63b7832ecd97\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"e694047d942a0f3cdf302b9730bd1a14cc7ec963be6452b90569e838482548c6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"51522ce3-b205-47c1-be53-086b8bd62ef3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a9c7c6d8ce8983c90c18d3fa7d38b1a9b9f954e52174c6a3d0c4710746720f96\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18489, \"end_char_idx\": 21603, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "In contrast to Flash Attention 1, the integration makes it possible to train your model on an arbitrary dataset that also includes padding tokens.\n\n\n### Using model creation utility\n\nWe included a utility function to create your model.\n\n[[autodoc]] ModelConfig\n\n```python\nfrom trl import ModelConfig, SFTTrainer, get_kbit_device_map, get_peft_config, get_quantization_config\nmodel_config = ModelConfig(\n    model_name_or_path=\"facebook/opt-350m\"\n    attn_implementation=None, # or \"flash_attention_2\"\n)\ntorch_dtype = (\n    model_config.torch_dtype\n    if model_config.torch_dtype in [\"auto\", None]\n    else getattr(torch, model_config.torch_dtype)\n)\nquantization_config = get_quantization_config(model_config)\nmodel_kwargs = dict(\n    revision=model_config.model_revision,\n    trust_remote_code=model_config.trust_remote_code,\n    attn_implementation=model_config.attn_implementation,\n    torch_dtype=torch_dtype,\n    use_cache=False if training_args.gradient_checkpointing else True,\n    device_map=get_kbit_device_map() if quantization_config is not None else None,\n    quantization_config=quantization_config,\n)\nmodel = AutoModelForCausalLM.from_pretrained(model_config.model_name_or_path, **model_kwargs)\ntrainer = SFTTrainer(\n    ...,\n    model=model_config.model_name_or_path,\n    peft_config=get_peft_config(model_config),\n)\n```\n\n### Enhance the model's performances using NEFTune\n\nNEFTune is a technique to boost the performance of chat models and was introduced by the paper [\"NEFTune: Noisy Embeddings Improve Instruction Finetuning\"](https://huggingface.co/papers/2310.05914) from Jain et al. it consists of adding noise to the embedding vectors during training. According to the abstract of the paper:\n\n>  Standard finetuning of LLaMA-2-7B using Alpaca achieves 29.79% on AlpacaEval, which rises to 64.69% using noisy embeddings. NEFTune also improves over strong baselines on modern instruction datasets. Models trained with Evol-Instruct see a 10% improvement, with ShareGPT an 8% improvement, and with OpenPlatypus an 8% improvement. Even powerful models further refined with RLHF such as LLaMA-2-Chat benefit from additional training with NEFTune.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/neft-screenshot.png\">\n</div>\n\nTo use it in `SFTTrainer` simply pass `neftune_noise_alpha` when creating your `SFTConfig` instance. Note that to avoid any surprising behaviour, NEFTune is disabled after training to retrieve back the original behaviour of the embedding layer.\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nsft_config = SFTConfig(\n    neftune_noise_alpha=5,\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\ntrainer.train()\n```\n\nWe have tested NEFTune by training `mistralai/Mistral-7B-v0.1` on the [OpenAssistant dataset](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) and validated that using NEFTune led to a performance boost of ~25% on MT Bench."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 51522ce3-b205-47c1-be53-086b8bd62ef3
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"51522ce3-b205-47c1-be53-086b8bd62ef3\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dbe0b07c-6ab9-4a73-a81d-2e38d094c7b1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"76f2fabf6ff5feb9ff373b94c6d7569302122941ecd091785ea561c4ea7aa235\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"68483bd2-751b-439a-994d-16af11f0edae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e4be16f78cf43baba0e7ebf3fc6cc616a955711dd487099a1d964cf7173d72a2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20093, \"end_char_idx\": 22339, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "it consists of adding noise to the embedding vectors during training. According to the abstract of the paper:\n\n>  Standard finetuning of LLaMA-2-7B using Alpaca achieves 29.79% on AlpacaEval, which rises to 64.69% using noisy embeddings. NEFTune also improves over strong baselines on modern instruction datasets. Models trained with Evol-Instruct see a 10% improvement, with ShareGPT an 8% improvement, and with OpenPlatypus an 8% improvement. Even powerful models further refined with RLHF such as LLaMA-2-Chat benefit from additional training with NEFTune.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/neft-screenshot.png\">\n</div>\n\nTo use it in `SFTTrainer` simply pass `neftune_noise_alpha` when creating your `SFTConfig` instance. Note that to avoid any surprising behaviour, NEFTune is disabled after training to retrieve back the original behaviour of the embedding layer.\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nsft_config = SFTConfig(\n    neftune_noise_alpha=5,\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\ntrainer.train()\n```\n\nWe have tested NEFTune by training `mistralai/Mistral-7B-v0.1` on the [OpenAssistant dataset](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) and validated that using NEFTune led to a performance boost of ~25% on MT Bench.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-neftune-mistral-7b.png\">\n</div>\n\nNote however, that the amount of performance gain is _dataset dependent_ and in particular, applying NEFTune on synthetic datasets like [UltraChat](https://huggingface.co/datasets/stingning/ultrachat) typically produces smaller gains.\n\n### Accelerate fine-tuning 2x using `unsloth`\n\nYou can further accelerate QLoRA / LoRA (2x faster, 60% less memory) using the [`unsloth`](https://github.com/unslothai/unsloth) library that is fully compatible with `SFTTrainer`. Currently `unsloth` supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 68483bd2-751b-439a-994d-16af11f0edae
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"68483bd2-751b-439a-994d-16af11f0edae\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"51522ce3-b205-47c1-be53-086b8bd62ef3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"e62e398576d49ff5bb8df5d8e70be2f077deb3de68a8391cab9b7b395dc68877\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5b7f22a0-b4f8-43ee-a644-8487746e088f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"15a3ccf58848003d1120e48953ba8d175514058488fad4c4eea6b50b65d4ccb8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20916, \"end_char_idx\": 23571, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Note that to avoid any surprising behaviour, NEFTune is disabled after training to retrieve back the original behaviour of the embedding layer.\n\n```python\nfrom datasets import load_dataset\nfrom trl import SFTConfig, SFTTrainer\n\ndataset = load_dataset(\"imdb\", split=\"train\")\n\nsft_config = SFTConfig(\n    neftune_noise_alpha=5,\n)\ntrainer = SFTTrainer(\n    \"facebook/opt-350m\",\n    train_dataset=dataset,\n    args=sft_config,\n)\ntrainer.train()\n```\n\nWe have tested NEFTune by training `mistralai/Mistral-7B-v0.1` on the [OpenAssistant dataset](https://huggingface.co/datasets/timdettmers/openassistant-guanaco) and validated that using NEFTune led to a performance boost of ~25% on MT Bench.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-neftune-mistral-7b.png\">\n</div>\n\nNote however, that the amount of performance gain is _dataset dependent_ and in particular, applying NEFTune on synthetic datasets like [UltraChat](https://huggingface.co/datasets/stingning/ultrachat) typically produces smaller gains.\n\n### Accelerate fine-tuning 2x using `unsloth`\n\nYou can further accelerate QLoRA / LoRA (2x faster, 60% less memory) using the [`unsloth`](https://github.com/unslothai/unsloth) library that is fully compatible with `SFTTrainer`. Currently `unsloth` supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures. Some benchmarks on 1x A100 listed below:\n\n| 1 A100 40GB     | Dataset   | \ud83e\udd17   | \ud83e\udd17 + Flash Attention 2 | \ud83e\udda5 Unsloth | \ud83e\udda5 VRAM saved |\n| --------------- | --------- | --- | --------------------- | --------- | ------------ |\n| Code Llama 34b  | Slim Orca | 1x  | 1.01x                 | **1.94x** | -22.7%       |\n| Llama-2 7b      | Slim Orca | 1x  | 0.96x                 | **1.87x** | -39.3%       |\n| Mistral 7b      | Slim Orca | 1x  | 1.17x                 | **1.88x** | -65.9%       |\n| Tiny Llama 1.1b | Alpaca    | 1x  | 1.55x                 | **2.74x** | -57.8%       |\n\nFirst install `unsloth` according to the [official documentation](https://github.com/unslothai/unsloth). Once installed, you can incorporate unsloth into your workflow in a very simple manner; instead of loading `AutoModelForCausalLM`, you just need to load a `FastLanguageModel` as follows:\n\n```python\nimport torch\nfrom trl import SFTConfig, SFTTrainer\nfrom unsloth import FastLanguageModel\n\nmax_seq_length = 2048 # Supports automatic RoPE Scaling, so choose any number\n\n# Load model\nmodel, tokenizer = FastLanguageModel.from_pretrained(\n    model_name=\"unsloth/mistral-7b\",\n    max_seq_length=max_seq_length,\n    dtype=None,  # None for auto detection."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5b7f22a0-b4f8-43ee-a644-8487746e088f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"5b7f22a0-b4f8-43ee-a644-8487746e088f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"68483bd2-751b-439a-994d-16af11f0edae\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"6affe4b8b11716772ae3988b733b8aab87d52504318e6387a8e6caa07ac0adaf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5cfee872-34a3-4996-aec6-78f0bc399e02\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"16b6d30a6d2c184efc21f2100de483797304561aa91fa724643ca25d10a9073f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22340, \"end_char_idx\": 25106, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Some benchmarks on 1x A100 listed below:\n\n| 1 A100 40GB     | Dataset   | \ud83e\udd17   | \ud83e\udd17 + Flash Attention 2 | \ud83e\udda5 Unsloth | \ud83e\udda5 VRAM saved |\n| --------------- | --------- | --- | --------------------- | --------- | ------------ |\n| Code Llama 34b  | Slim Orca | 1x  | 1.01x                 | **1.94x** | -22.7%       |\n| Llama-2 7b      | Slim Orca | 1x  | 0.96x                 | **1.87x** | -39.3%       |\n| Mistral 7b      | Slim Orca | 1x  | 1.17x                 | **1.88x** | -65.9%       |\n| Tiny Llama 1.1b | Alpaca    | 1x  | 1.55x                 | **2.74x** | -57.8%       |\n\nFirst install `unsloth` according to the [official documentation](https://github.com/unslothai/unsloth). Once installed, you can incorporate unsloth into your workflow in a very simple manner; instead of loading `AutoModelForCausalLM`, you just need to load a `FastLanguageModel` as follows:\n\n```python\nimport torch\nfrom trl import SFTConfig, SFTTrainer\nfrom unsloth import FastLanguageModel\n\nmax_seq_length = 2048 # Supports automatic RoPE Scaling, so choose any number\n\n# Load model\nmodel, tokenizer = FastLanguageModel.from_pretrained(\n    model_name=\"unsloth/mistral-7b\",\n    max_seq_length=max_seq_length,\n    dtype=None,  # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n    load_in_4bit=True,  # Use 4bit quantization to reduce memory usage. Can be False\n    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n)\n\n# Do model patching and add fast LoRA weights\nmodel = FastLanguageModel.get_peft_model(\n    model,\n    r=16,\n    target_modules=[\n        \"q_proj\",\n        \"k_proj\",\n        \"v_proj\",\n        \"o_proj\",\n        \"gate_proj\",\n        \"up_proj\",\n        \"down_proj\",\n    ],\n    lora_alpha=16,\n    lora_dropout=0,  # Dropout = 0 is currently optimized\n    bias=\"none\",  # Bias = \"none\" is currently optimized\n    use_gradient_checkpointing=True,\n    random_state=3407,\n)\n\nargs = SFTConfig(\n    output_dir=\"./output\",\n    max_seq_length=max_seq_length,\n    dataset_text_field=\"text\",\n)\n\ntrainer = SFTTrainer(\n    model=model,\n    args=args,\n    train_dataset=dataset,\n)\ntrainer.train()\n```\n\nThe saved model is fully compatible with Hugging Face's transformers library. Learn more about unsloth in their [official repository](https://github.com/unslothai/unsloth).\n\n## Best practices\n\nPay attention to the following best practices when training a model with that trainer:\n\n- [`SFTTrainer`] always pads by default the sequences to the `max_seq_length` argument of the [`SFTTrainer`]. If none is passed, the trainer will retrieve that value from the tokenizer. Some tokenizers do not provide a default value, so there is a check to retrieve the minimum between 2048 and that value. Make sure to check it before training."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5cfee872-34a3-4996-aec6-78f0bc399e02
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"5cfee872-34a3-4996-aec6-78f0bc399e02\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5b7f22a0-b4f8-43ee-a644-8487746e088f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"cea1b043be7f15de241988b38779e11ac4b7343a919b4e0e92bd7cc5cdc1b0ce\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"91c553ac-e9f5-41a3-b14f-1875c83c04f8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"69becf898bdfe2aae9c4312adeb4028a9963ade0d148dc9e0a135b1efac30af6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23572, \"end_char_idx\": 26308, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n    load_in_4bit=True,  # Use 4bit quantization to reduce memory usage. Can be False\n    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n)\n\n# Do model patching and add fast LoRA weights\nmodel = FastLanguageModel.get_peft_model(\n    model,\n    r=16,\n    target_modules=[\n        \"q_proj\",\n        \"k_proj\",\n        \"v_proj\",\n        \"o_proj\",\n        \"gate_proj\",\n        \"up_proj\",\n        \"down_proj\",\n    ],\n    lora_alpha=16,\n    lora_dropout=0,  # Dropout = 0 is currently optimized\n    bias=\"none\",  # Bias = \"none\" is currently optimized\n    use_gradient_checkpointing=True,\n    random_state=3407,\n)\n\nargs = SFTConfig(\n    output_dir=\"./output\",\n    max_seq_length=max_seq_length,\n    dataset_text_field=\"text\",\n)\n\ntrainer = SFTTrainer(\n    model=model,\n    args=args,\n    train_dataset=dataset,\n)\ntrainer.train()\n```\n\nThe saved model is fully compatible with Hugging Face's transformers library. Learn more about unsloth in their [official repository](https://github.com/unslothai/unsloth).\n\n## Best practices\n\nPay attention to the following best practices when training a model with that trainer:\n\n- [`SFTTrainer`] always pads by default the sequences to the `max_seq_length` argument of the [`SFTTrainer`]. If none is passed, the trainer will retrieve that value from the tokenizer. Some tokenizers do not provide a default value, so there is a check to retrieve the minimum between 2048 and that value. Make sure to check it before training.\n- For training adapters in 8bit, you might need to tweak the arguments of the `prepare_model_for_kbit_training` method from PEFT, hence we advise users to use `prepare_in_int8_kwargs` field, or create the `PeftModel` outside the [`SFTTrainer`] and pass it.\n- For a more memory-efficient training using adapters, you can load the base model in 8bit, for that simply add `load_in_8bit` argument when creating the [`SFTTrainer`], or create a base model in 8bit outside the trainer and pass it.\n- If you create a model outside the trainer, make sure to not pass to the trainer any additional keyword arguments that are relative to `from_pretrained()` method.\n\n## Multi-GPU Training\n\nTrainer (and thus SFTTrainer) supports multi-GPU training. If you run your script with `python script.py` it will default to using DP as the strategy, which may be [slower than expected](https://github.com/huggingface/trl/issues/1303). To use DDP (which is generally recommended, see [here](https://huggingface.co/docs/transformers/en/perf_train_gpu_many?select-gpu=Accelerate#data-parallelism) for more info) you must launch the script with `python -m torch.distributed.launch script.py` or `accelerate launch script.py`."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 91c553ac-e9f5-41a3-b14f-1875c83c04f8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"91c553ac-e9f5-41a3-b14f-1875c83c04f8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5cfee872-34a3-4996-aec6-78f0bc399e02\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"1cf6711a2f59a3e21020816a10aecbe71fc5fee4a7afe2b6324b2dc033dbed0a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"676735f0-779b-4db8-868c-d930130ec3bb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f9d4ab76c5d403ac01de20b5db695f9fd813635544dec04dd7968596ba247c1e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24871, \"end_char_idx\": 28020, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "If none is passed, the trainer will retrieve that value from the tokenizer. Some tokenizers do not provide a default value, so there is a check to retrieve the minimum between 2048 and that value. Make sure to check it before training.\n- For training adapters in 8bit, you might need to tweak the arguments of the `prepare_model_for_kbit_training` method from PEFT, hence we advise users to use `prepare_in_int8_kwargs` field, or create the `PeftModel` outside the [`SFTTrainer`] and pass it.\n- For a more memory-efficient training using adapters, you can load the base model in 8bit, for that simply add `load_in_8bit` argument when creating the [`SFTTrainer`], or create a base model in 8bit outside the trainer and pass it.\n- If you create a model outside the trainer, make sure to not pass to the trainer any additional keyword arguments that are relative to `from_pretrained()` method.\n\n## Multi-GPU Training\n\nTrainer (and thus SFTTrainer) supports multi-GPU training. If you run your script with `python script.py` it will default to using DP as the strategy, which may be [slower than expected](https://github.com/huggingface/trl/issues/1303). To use DDP (which is generally recommended, see [here](https://huggingface.co/docs/transformers/en/perf_train_gpu_many?select-gpu=Accelerate#data-parallelism) for more info) you must launch the script with `python -m torch.distributed.launch script.py` or `accelerate launch script.py`. For DDP to work you must also check the following:\n- If you're using gradient_checkpointing, add the following to the TrainingArguments: `gradient_checkpointing_kwargs={'use_reentrant':False}` (more info [here](https://github.com/huggingface/transformers/issues/26969)\n- Ensure that the model is placed on the correct device:\n```python\nfrom accelerate import PartialState\ndevice_string = PartialState().process_index\nmodel = AutoModelForCausalLM.from_pretrained(\n     ...\n    device_map={'':device_string}\n)\n```\n\n## GPTQ Conversion\n\nYou may experience some issues with GPTQ Quantization after completing training. Lowering `gradient_accumulation_steps` to `4` will resolve most issues during the quantization process to GPTQ format.\n\n## Extending `SFTTrainer` for Vision Language Models\n\n`SFTTrainer` does not inherently support vision-language data. However, we provide a guide on how to tweak the trainer to support vision-language data. Specifically, you need to use a custom data collator that is compatible with vision-language data. This guide outlines the steps to make these adjustments. For a concrete example, refer to the script [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py) which demonstrates how to fine-tune the LLaVA 1.5 model on the [HuggingFaceH4/llava-instruct-mix-vsft](https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft) dataset.\n\n### Preparing the Data\n\nThe data format is flexible, provided it is compatible with the custom collator that we will define later. A common approach is to use conversational data. Given that the data includes both text and images, the format needs to be adjusted accordingly."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 676735f0-779b-4db8-868c-d930130ec3bb
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"676735f0-779b-4db8-868c-d930130ec3bb\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"91c553ac-e9f5-41a3-b14f-1875c83c04f8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"05591c0404fab6c7eba7bc478419db37b6881c0b85209bad15bb043f50870ac9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cf6cb03d-c93a-476f-a2a5-107908ae6507\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5872c6f8c848ca5cbd08574b026f0eb754fa77a62c1dbee55ba0932fc1ec8a61\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 26923, \"end_char_idx\": 29792, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Lowering `gradient_accumulation_steps` to `4` will resolve most issues during the quantization process to GPTQ format.\n\n## Extending `SFTTrainer` for Vision Language Models\n\n`SFTTrainer` does not inherently support vision-language data. However, we provide a guide on how to tweak the trainer to support vision-language data. Specifically, you need to use a custom data collator that is compatible with vision-language data. This guide outlines the steps to make these adjustments. For a concrete example, refer to the script [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py) which demonstrates how to fine-tune the LLaVA 1.5 model on the [HuggingFaceH4/llava-instruct-mix-vsft](https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft) dataset.\n\n### Preparing the Data\n\nThe data format is flexible, provided it is compatible with the custom collator that we will define later. A common approach is to use conversational data. Given that the data includes both text and images, the format needs to be adjusted accordingly. Below is an example of a conversational data format involving both text and images:\n\n```python\nimages = [\"obama.png\"]\nmessages = [\n    {\n        \"role\": \"user\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"Who is this?\"},\n            {\"type\": \"image\"}\n        ]\n    },\n    {\n        \"role\": \"assistant\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"Barack Obama\"}\n        ]\n    },\n    {\n        \"role\": \"user\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"What is he famous for?\"}\n        ]\n    },\n    {\n        \"role\": \"assistant\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"He is the 44th President of the United States.\"}\n        ]\n    }\n]\n```\n\nTo illustrate how this data format will be processed using the LLaVA model, you can use the following code:\n\n```python\nfrom transformers import AutoProcessor\n\nprocessor = AutoProcessor.from_pretrained(\"llava-hf/llava-1.5-7b-hf\")\nprint(processor.apply_chat_template(messages, tokenize=False))\n```\n\nThe output will be formatted as follows:\n\n```txt\nWho is this? ASSISTANT: Barack Obama USER: What is he famous for? ASSISTANT: He is the 44th President of the United States. \n```\n\n<iframe src=\"https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft/embed/viewer/default/train\" frameborder=\"0\" width=\"100%\" height=\"560px\"></iframe>\n\n\n### A custom collator for processing multi-modal data\n\nUnlike the default behavior of `SFTTrainer`, processing multi-modal data is done on the fly during the data collation process. To do this, you need to define a custom collator that processes both the text and images. This collator must take a list of examples as input (see the previous section for an example of the data format) and return a batch of processed data."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: cf6cb03d-c93a-476f-a2a5-107908ae6507
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"cf6cb03d-c93a-476f-a2a5-107908ae6507\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"676735f0-779b-4db8-868c-d930130ec3bb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"ac0ae4adb51636ac41488aac5771fe3a0e2f243f8ae27aa70d0a244769237475\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"248d91f3-fd29-4cf4-8812-4020e8ec8b65\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b5dbb979e1f532cf5e354845e361c9e313006f0a2137ab7e12ba8fa8c3f1180a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 28253, \"end_char_idx\": 31366, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "\"},\n            {\"type\": \"image\"}\n        ]\n    },\n    {\n        \"role\": \"assistant\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"Barack Obama\"}\n        ]\n    },\n    {\n        \"role\": \"user\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"What is he famous for?\"}\n        ]\n    },\n    {\n        \"role\": \"assistant\",\n        \"content\": [\n            {\"type\": \"text\", \"text\": \"He is the 44th President of the United States.\"}\n        ]\n    }\n]\n```\n\nTo illustrate how this data format will be processed using the LLaVA model, you can use the following code:\n\n```python\nfrom transformers import AutoProcessor\n\nprocessor = AutoProcessor.from_pretrained(\"llava-hf/llava-1.5-7b-hf\")\nprint(processor.apply_chat_template(messages, tokenize=False))\n```\n\nThe output will be formatted as follows:\n\n```txt\nWho is this? ASSISTANT: Barack Obama USER: What is he famous for? ASSISTANT: He is the 44th President of the United States. \n```\n\n<iframe src=\"https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft/embed/viewer/default/train\" frameborder=\"0\" width=\"100%\" height=\"560px\"></iframe>\n\n\n### A custom collator for processing multi-modal data\n\nUnlike the default behavior of `SFTTrainer`, processing multi-modal data is done on the fly during the data collation process. To do this, you need to define a custom collator that processes both the text and images. This collator must take a list of examples as input (see the previous section for an example of the data format) and return a batch of processed data. Below is an example of such a collator:\n\n```python\ndef collate_fn(examples):\n    # Get the texts and images, and apply the chat template\n    texts = [processor.apply_chat_template(example[\"messages\"], tokenize=False) for example in examples]\n    images = [example[\"images\"][0] for example in examples]\n\n    # Tokenize the texts and process the images\n    batch = processor(texts, images, return_tensors=\"pt\", padding=True)\n\n    # The labels are the input_ids, and we mask the padding tokens in the loss computation\n    labels = batch[\"input_ids\"].clone()\n    labels[labels == processor.tokenizer.pad_token_id] = -100\n    batch[\"labels\"] = labels\n\n    return batch\n```\n\nWe can verify that the collator works as expected by running the following code:\n\n```python\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"HuggingFaceH4/llava-instruct-mix-vsft\", split=\"train\")\nexamples = [dataset[0], dataset[1]]  # Just two examples for the sake of the example\ncollated_data = collate_fn(examples)\nprint(collated_data.keys())  # dict_keys(['input_ids', 'attention_mask', 'pixel_values', 'labels'])\n```\n\n### Training the vision-language model\n\nNow that we have prepared the data and defined the collator, we can proceed with training the model. To ensure that the data is not processed as text-only, we need to set a couple of arguments in the `SFTConfig`, specifically `dataset_text_field` and `remove_unused_columns`. We also need to set `skip_prepare_dataset` to `True` to avoid the default processing of the dataset. Below is an example of how to set up the `SFTTrainer`."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 248d91f3-fd29-4cf4-8812-4020e8ec8b65
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"248d91f3-fd29-4cf4-8812-4020e8ec8b65\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cf6cb03d-c93a-476f-a2a5-107908ae6507\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"e71b25446d076ebe99d9c0a71fe64b07a2ea5430cbc18993037b6496d5ac3773\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ad816c64-14d7-4bbd-b3e1-b1d1d7dd5998\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0377f2a529f33e386ed29da47e143213e0af65dcf414097d7bd59b1412725f24\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 29793, \"end_char_idx\": 32782, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "Below is an example of such a collator:\n\n```python\ndef collate_fn(examples):\n    # Get the texts and images, and apply the chat template\n    texts = [processor.apply_chat_template(example[\"messages\"], tokenize=False) for example in examples]\n    images = [example[\"images\"][0] for example in examples]\n\n    # Tokenize the texts and process the images\n    batch = processor(texts, images, return_tensors=\"pt\", padding=True)\n\n    # The labels are the input_ids, and we mask the padding tokens in the loss computation\n    labels = batch[\"input_ids\"].clone()\n    labels[labels == processor.tokenizer.pad_token_id] = -100\n    batch[\"labels\"] = labels\n\n    return batch\n```\n\nWe can verify that the collator works as expected by running the following code:\n\n```python\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"HuggingFaceH4/llava-instruct-mix-vsft\", split=\"train\")\nexamples = [dataset[0], dataset[1]]  # Just two examples for the sake of the example\ncollated_data = collate_fn(examples)\nprint(collated_data.keys())  # dict_keys(['input_ids', 'attention_mask', 'pixel_values', 'labels'])\n```\n\n### Training the vision-language model\n\nNow that we have prepared the data and defined the collator, we can proceed with training the model. To ensure that the data is not processed as text-only, we need to set a couple of arguments in the `SFTConfig`, specifically `dataset_text_field` and `remove_unused_columns`. We also need to set `skip_prepare_dataset` to `True` to avoid the default processing of the dataset. Below is an example of how to set up the `SFTTrainer`.\n\n```python\nargs.dataset_text_field = \"\"  # needs a dummy field\nargs.remove_unused_columns = False\nargs.dataset_kwargs = {\"skip_prepare_dataset\": True}\n\ntrainer = SFTTrainer(\n    model=model,\n    args=args,\n    data_collator=collate_fn,\n    train_dataset=train_dataset,\n    tokenizer=processor.tokenizer,\n)\n```\n\nA full example of training LLaVa 1.5 on the [HuggingFaceH4/llava-instruct-mix-vsft](https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft) dataset can be found in the script [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py).\n\n- [Experiment tracking](https://wandb.ai/huggingface/trl/runs/2b2c5l7s)\n- [Trained model](https://huggingface.co/HuggingFaceH4/sft-llava-1.5-7b-hf)\n\n## SFTTrainer\n\n[[autodoc]] SFTTrainer\n\n## SFTConfig\n\n[[autodoc]] SFTConfig\n\n## Datasets\n\nIn the SFTTrainer we smartly support `datasets.IterableDataset` in addition to other style datasets. This is useful if you are using large corpora that you do not want to save all to disk. The data will be tokenized and processed on the fly, even when packing is enabled.\n\nAdditionally, in the SFTTrainer, we support pre-tokenized datasets if they are `datasets.Dataset` or `datasets.IterableDataset`. In other words, if such a dataset has a column of `input_ids`, no further processing (tokenization or packing) will be done, and the dataset will be used as-is."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: ad816c64-14d7-4bbd-b3e1-b1d1d7dd5998
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sft_trainer", "title": "Supervised Fine-tuning Trainer", "tokens": 8125, "retrieve_doc": false, "source": "trl", "_node_content": "{\"id_\": \"ad816c64-14d7-4bbd-b3e1-b1d1d7dd5998\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ae2c8951-dc4a-4f1e-bb99-8b8f9051d504\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"7923fb4559a2194ab736a7dc8d37ae34ce6c85f6fa69571bf8b3cb04b2fe3cef\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"248d91f3-fd29-4cf4-8812-4020e8ec8b65\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sft_trainer\", \"title\": \"Supervised Fine-tuning Trainer\", \"tokens\": 8125, \"retrieve_doc\": false, \"source\": \"trl\"}, \"hash\": \"475ce37299444434f4d0a81bc8aed94d6dd6d42dd15a0074531df1cfe2ad0988\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 31368, \"end_char_idx\": 32964, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "ref_doc_id": "ae2c8951-dc4a-4f1e-bb99-8b8f9051d504", "chroma:document": "```python\nargs.dataset_text_field = \"\"  # needs a dummy field\nargs.remove_unused_columns = False\nargs.dataset_kwargs = {\"skip_prepare_dataset\": True}\n\ntrainer = SFTTrainer(\n    model=model,\n    args=args,\n    data_collator=collate_fn,\n    train_dataset=train_dataset,\n    tokenizer=processor.tokenizer,\n)\n```\n\nA full example of training LLaVa 1.5 on the [HuggingFaceH4/llava-instruct-mix-vsft](https://huggingface.co/datasets/HuggingFaceH4/llava-instruct-mix-vsft) dataset can be found in the script [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py).\n\n- [Experiment tracking](https://wandb.ai/huggingface/trl/runs/2b2c5l7s)\n- [Trained model](https://huggingface.co/HuggingFaceH4/sft-llava-1.5-7b-hf)\n\n## SFTTrainer\n\n[[autodoc]] SFTTrainer\n\n## SFTConfig\n\n[[autodoc]] SFTConfig\n\n## Datasets\n\nIn the SFTTrainer we smartly support `datasets.IterableDataset` in addition to other style datasets. This is useful if you are using large corpora that you do not want to save all to disk. The data will be tokenized and processed on the fly, even when packing is enabled.\n\nAdditionally, in the SFTTrainer, we support pre-tokenized datasets if they are `datasets.Dataset` or `datasets.IterableDataset`. In other words, if such a dataset has a column of `input_ids`, no further processing (tokenization or packing) will be done, and the dataset will be used as-is. This can be useful if you have pretokenized your dataset outside of this script and want to re-use it directly.\n\n### ConstantLengthDataset\n\n[[autodoc]] trainer.ConstantLengthDataset"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 999ee9d8-50c3-46c0-8e04-41cce764acce
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"999ee9d8-50c3-46c0-8e04-41cce764acce\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"732220fc-ebf5-4951-999b-237b39b34293\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"237de08958cda6d799152881bcd456079257d4cc02229060d5a0c51436d84c9d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3247, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "# PPOv2 Trainer\n\nTRL supports training LLMs with [Proximal Policy Optimization (PPO)](https://huggingface.co/papers/1707.06347).\n\nReferences:\n- [Fine-Tuning Language Models from Human Preferences](https://github.com/openai/lm-human-preferences)\n- [Learning to Summarize from Human Feedback](https://github.com/openai/summarize-from-feedback)\n- [The N Implementation Details of RLHF with PPO](https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo)\n- [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031)\n\n## Get started\n\nTo just run a PPO script to make sure the trainer can run, you can run the following command to train a PPO model with a dummy reward model.\n\n```bash\npython examples/scripts/ppo/ppo.py \\\n    --learning_rate 3e-6 \\\n    --num_ppo_epochs 1 \\\n    --num_mini_batches 1 \\\n    --output_dir models/minimal/ppo \\\n    --per_device_train_batch_size 64 \\\n    --gradient_accumulation_steps 1 \\\n    --total_episodes 10000 \\\n    --model_name_or_path EleutherAI/pythia-1b-deduped \\\n    --non_eos_penalty\n```\n\n\n## Explanation of the logged metrics\n\nThe logged metrics are as follows. Here is an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35)\n\n* `eps`: Tracks the number of episodes per second.\n* `objective/kl`: The mean Kullback-Leibler (KL) divergence between the current policy and reference policy.\n* `objective/entropy`: The mean entropy of the policy, indicating the randomness of the actions chosen by the policy.\n* `objective/non_score_reward`: The mean reward from non-score-related sources, basically `beta * kl.sum(1)`, where `beta` is the KL penalty coefficient and `kl` is the per-token KL divergence.\n* `objective/rlhf_reward`: The mean RLHF reward, which is `score - non_score_reward`.\n* `objective/scores`: The mean scores returned by the reward model / environment.\n* `policy/approxkl_avg`: The average approximate KL divergence between consecutive PPO policies. Note that this is not the same as `objective/kl`.\n* `policy/clipfrac_avg`: The average fraction of policy updates that are clipped, indicating how often the policy updates are constrained to prevent large changes.\n* `loss/policy_avg`: The average policy loss, indicating how well the policy is performing.\n* `loss/value_avg`: The average value loss, indicating the difference between the predicted value and the actual reward.\n* `val/clipfrac_avg`: The average fraction of value function updates that are clipped, similar to policy/clipfrac_avg but for the value function.\n* `policy/entropy_avg`: The average entropy of the policy during training, indicating how diverse the policy's actions are.\n* `val/ratio`: The mean ratio of the current policy probability to the old policy probability, providing a measure of how much the policy has changed.\n* `val/ratio_var`: The variance of the `val/ratio`, indicating the variability in policy changes.\n* `val/num_eos_tokens`: The number of end-of-sequence (EOS) tokens generated, which can indicate the number of complete responses.\n* `lr`: lr: The current learning rate used by the optimizer.\n* `episode`: episode: The current global step or episode count in the training process."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 732220fc-ebf5-4951-999b-237b39b34293
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"732220fc-ebf5-4951-999b-237b39b34293\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"999ee9d8-50c3-46c0-8e04-41cce764acce\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"475dd759e3b9550bc8d0e97fbc8b73209c248d474777d7efd3d57b69e6a382b7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cb34e016-5c05-42fd-9503-ff82873d2e12\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a15e25646b7c15b6d7e0e0205e85014d4c5074e67b488c95f4d0faa98345ae0e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3250, \"end_char_idx\": 6197, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "## Cookbook\n\n* Debugging TIP: `objective/rlhf_reward`: this is the ultimate objective of the RLHF training. If training works as intended, this metric should keep going up.\n* Debugging TIP: `val/ratio`: this number should float around 1.0, and it gets clipped by `--cliprange 0.2` with PPO's surrogate loss. So if this `ratio` is too high like 2.0 or 1000.0 or too small like 0.1, it means the updates between consecutive policies are too drastic. You should try undertand why this is happening and try to fix it.\n* Memory TIP: If you are running out of memory, you can try to reduce the `--per_device_train_batch_size` or increase the `--gradient_accumulation_steps` to reduce the memory footprint.\n* Memory TIP: If you have multiple GPUs, you can also run training with DeepSpeed stage 3 to reduce the memory footprint `accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml`.\n* Usage TIP: We recommend to use the \"EOS trick\" via `--non_eos_penalty --stop_token eos`, which replaces the score of completions that do not end with an EOS token with a static scalar penalty `--penalty_reward_value`. This can help the model learn to generate more coherent completions.\n\n\n## What is my model doing exactly?\n\nTo help you understand what your model is doing, we periodically log some sample completions from the model. Here is an example of a completion. In an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/dd2o3g35), it looks like the following, allowing you to see the model's response at different stages of training. By default we generate `--num_sample_generations 10` during training, but you can customize the number of generations.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/ppov2_completions.gif?download=true)\n\n\nIn the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?               \u2502 desperate.<|endoftext|>[PAD][P\u2026 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 POST: Hi,                       \u2502                                 \u2502          \u2502\n\u2502 I'm 22, and I have been with my \u2502                                 \u2502          \u2502\n\u2502 girlfriend for 5 years now. We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: cb34e016-5c05-42fd-9503-ff82873d2e12
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"cb34e016-5c05-42fd-9503-ff82873d2e12\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"732220fc-ebf5-4951-999b-237b39b34293\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"87069cc1709753a5b5324447f5047c8e892501791f4e848ba51191f44256ae27\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a5b940ee-7a27-4b39-8cb0-da0451602333\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"30a60f1fce9c5beba5db830dfeae9ab17ec87e98143f953a48aeb86e5ec03365\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5080, \"end_char_idx\": 8816, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "In the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?               \u2502 desperate.<|endoftext|>[PAD][P\u2026 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 POST: Hi,                       \u2502                                 \u2502          \u2502\n\u2502 I'm 22, and I have been with my \u2502                                 \u2502          \u2502\n\u2502 girlfriend for 5 years now. We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Problem, I recently started to  \u2502                                 \u2502          \u2502\n\u2502 have feelings for an other      \u2502                                 \u2502          \u2502\n\u2502 person (a friend). This person  \u2502                                 \u2502          \u2502\n\u2502 has had a boyfriend for now 3   \u2502                                 \u2502          \u2502\n\u2502 years, and has absolutely no    \u2502                                 \u2502          \u2502\n\u2502 ideas. Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a5b940ee-7a27-4b39-8cb0-da0451602333
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a5b940ee-7a27-4b39-8cb0-da0451602333\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cb34e016-5c05-42fd-9503-ff82873d2e12\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0d32ff243eabab76ae5369da50b0f17e59b101c9ea289ea98f8def350b8ddd5a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7573cba2-79d3-4822-9831-523bbac5f7e9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"31586958e3153923b93a38875e999a53ff30cb5b489924b315a66b839e294cb7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6761, \"end_char_idx\": 10582, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV. I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down. She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 7573cba2-79d3-4822-9831-523bbac5f7e9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"7573cba2-79d3-4822-9831-523bbac5f7e9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a5b940ee-7a27-4b39-8cb0-da0451602333\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"428ae662230939133dd59c5c9090254d4739f451d74cf38aa4cf17ce8f55ccd2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"95014e62-6f90-48a1-bc81-ce218ba89d50\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8f245e7466d787f4c4e7ce0bc090e4ba70292ec758801c57616b357f6ffe581b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8817, \"end_char_idx\": 12867, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down. She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality. I had no  \u2502                                 \u2502          \u2502\n\u2502 lock on my door; but I had a    \u2502                                 \u2502          \u2502\n\u2502 dresser right next to it, so I  \u2502                                 \u2502          \u2502\n\u2502 pulled one of the drawers out   \u2502                                 \u2502          \u2502\n\u2502 enough so that it caused the    \u2502                                 \u2502          \u2502\n\u2502 door to not be openable. Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it. But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nThis PPOv2 implementation is based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 95014e62-6f90-48a1-bc81-ce218ba89d50
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"95014e62-6f90-48a1-bc81-ce218ba89d50\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7573cba2-79d3-4822-9831-523bbac5f7e9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"23c41330e4974ae23a246c9bace01bc2290707495a1b56a287599c3fab47131b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"354717bd-39cf-4d8a-a84b-270cb7490eb1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6a1243cd8649363a5c2e028f703bf80bcadb8c71b4716a3bbec3ce1a83fa3887\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10991, \"end_char_idx\": 14240, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it. But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nThis PPOv2 implementation is based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n## Benchmark experiments\n\nTo validate the PPO implementation works, we ran experiment on the 1B model. Here are the command we used to run the experiment. We take the SFT / RM models directly from [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n```\naccelerate launch --config_file examples/accelerate_configs/deepspeed_zero2.yaml \\\n    examples/scripts/ppo/ppo_tldr.py \\\n    --output_dir models/minimal/ppo_tldr \\\n    --learning_rate 3e-6 \\\n    --per_device_train_batch_size 16 \\\n    --gradient_accumulation_steps 4 \\\n    --total_episodes 1000000 \\\n    --model_name_or_path EleutherAI/pythia-1b-deduped \\\n    --sft_model_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr \\\n    --reward_model_path cleanrl/EleutherAI_pythia-1b-deduped__reward__tldr \\\n    --local_rollout_forward_batch_size 16 \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n```\n\nCheckpoints and experiment tracking are available at:\n\n- [\ud83e\udd17 Model checkpoint](https://huggingface.co/vwxyzjn/ppo_tldr)\n- [\ud83d\udc1d Tracked experiment](https://wandb.ai/huggingface/trl/runs/dd2o3g35)\n\nTo evaluate, we use [vLLM](https://github.com/vllm-project/vllm) to load the checkpoints and GPT-4o mini as a judge model to evaluate the generated TL;DR against the reference TL;DR.\nFor more information on how to use judges, see [Judges](judges)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 354717bd-39cf-4d8a-a84b-270cb7490eb1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"354717bd-39cf-4d8a-a84b-270cb7490eb1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"95014e62-6f90-48a1-bc81-ce218ba89d50\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"912b7cf9e2555c4af5632393350291493373e59748d2850a6a059df10f083414\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d5b667fc-22c5-4ea3-bd74-1daef3403c38\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f9e994b7aac352611cd06c0b579d9dff9eb71e3a3b8f50880d3cb04cae73406c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12869, \"end_char_idx\": 14888, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "## Benchmark experiments\n\nTo validate the PPO implementation works, we ran experiment on the 1B model. Here are the command we used to run the experiment. We take the SFT / RM models directly from [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n```\naccelerate launch --config_file examples/accelerate_configs/deepspeed_zero2.yaml \\\n    examples/scripts/ppo/ppo_tldr.py \\\n    --output_dir models/minimal/ppo_tldr \\\n    --learning_rate 3e-6 \\\n    --per_device_train_batch_size 16 \\\n    --gradient_accumulation_steps 4 \\\n    --total_episodes 1000000 \\\n    --model_name_or_path EleutherAI/pythia-1b-deduped \\\n    --sft_model_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr \\\n    --reward_model_path cleanrl/EleutherAI_pythia-1b-deduped__reward__tldr \\\n    --local_rollout_forward_batch_size 16 \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n```\n\nCheckpoints and experiment tracking are available at:\n\n- [\ud83e\udd17 Model checkpoint](https://huggingface.co/vwxyzjn/ppo_tldr)\n- [\ud83d\udc1d Tracked experiment](https://wandb.ai/huggingface/trl/runs/dd2o3g35)\n\nTo evaluate, we use [vLLM](https://github.com/vllm-project/vllm) to load the checkpoints and GPT-4o mini as a judge model to evaluate the generated TL;DR against the reference TL;DR.\nFor more information on how to use judges, see [Judges](judges).\n\n```bash\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 33.00%\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/ppo_tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 64.70%\n```\n\nThe PPO checkpoint gets a 64.7% preferred rate vs the 33.0% preference rate of the SFT checkpoint. This is a good sign that the PPO training is working as intended.\n\nMetrics:\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/pr-1540/ppov2.png)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d5b667fc-22c5-4ea3-bd74-1daef3403c38
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppov2_trainer", "title": "PPOv2 Trainer", "tokens": 3660, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d5b667fc-22c5-4ea3-bd74-1daef3403c38\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95309df1-1642-4770-9274-8f1163b01657\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4cf9503e2f19728bbbbc3a1572f3867861d7ebf6b9ba72bc5cfd274cef78789f\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"354717bd-39cf-4d8a-a84b-270cb7490eb1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppov2_trainer\", \"title\": \"PPOv2 Trainer\", \"tokens\": 3660, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"ce709026f3e0b15bc2619e125c02498adfb510fbd95308c95c9204ba9848b29b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14176, \"end_char_idx\": 16008, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95309df1-1642-4770-9274-8f1163b01657", "doc_id": "95309df1-1642-4770-9274-8f1163b01657", "ref_doc_id": "95309df1-1642-4770-9274-8f1163b01657", "chroma:document": "For more information on how to use judges, see [Judges](judges).\n\n```bash\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 33.00%\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/ppo_tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 64.70%\n```\n\nThe PPO checkpoint gets a 64.7% preferred rate vs the 33.0% preference rate of the SFT checkpoint. This is a good sign that the PPO training is working as intended.\n\nMetrics:\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/pr-1540/ppov2.png)\n\n\n```bash\n# pip install openrlbenchmark==0.2.1a5\n# see https://github.com/openrlbenchmark/openrlbenchmark#get-started for documentation\n# to use it, change `?we=huggingface&wpn=trl` to your own project and `?tag=pr-1540` to your own tag\npython -m openrlbenchmark.rlops_multi_metrics \\\n    --filters '?we=huggingface&wpn=trl&xaxis=train/episode&ceik=output_dir&cen=sft_model_path&metrics=train/objective/rlhf_reward&metrics=train/objective/scores&metrics=train/objective/kl&metrics=train/objective/non_score_reward&metrics=train/objective/entropy&metrics=train/policy/approxkl_avg&metrics=train/policy/clipfrac_avg&metrics=train/loss/policy_avg&metrics=train/loss/value_avg&metrics=train/val/clipfrac_avg&metrics=train/policy/entropy_avg&metrics=train/val/ratio&metrics=train/val/ratio_var&metrics=train/val/num_eos_tokens&metrics=train/lr&metrics=train/eps' \\\n        \"cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr?tag=pr-1540\" \\\n    --env-ids models/minimal/ppo_tldr \\\n    --pc.ncols 4 \\\n    --pc.ncols-legend 1 \\\n    --pc.xlabel \"Episode\" \\\n    --output-filename benchmark/trl/pr-1540/ppov2 \\\n    --scan-history\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b3fa3836-5406-46bd-a5af-31027f311bcd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/iterative_sft_trainer", "title": "Iterative Trainer", "tokens": 282, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b3fa3836-5406-46bd-a5af-31027f311bcd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/iterative_sft_trainer\", \"title\": \"Iterative Trainer\", \"tokens\": 282, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93879eff-40f1-422d-952a-802f15c43d0e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/iterative_sft_trainer\", \"title\": \"Iterative Trainer\", \"tokens\": 282, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"71796ff47d6ecbd76e1720fbdb64e01236dba1a86fd041a536d5448b642bdaeb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1249, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93879eff-40f1-422d-952a-802f15c43d0e", "doc_id": "93879eff-40f1-422d-952a-802f15c43d0e", "ref_doc_id": "93879eff-40f1-422d-952a-802f15c43d0e", "chroma:document": "# Iterative Trainer\n\nIterative fine-tuning is a training method that enables to perform custom actions (generation and filtering for example) between optimization steps. In TRL we provide an easy-to-use API to fine-tune your models in an iterative way in just a few lines of code.\n\n## Usage\n\nTo get started quickly, instantiate an instance a model, and a tokenizer.\n\n```python\n\nmodel = AutoModelForCausalLM.from_pretrained(model_name)\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nif tokenizer.pad_token is None:\n    tokenizer.pad_token = tokenizer.eos_token\n\ntrainer = IterativeSFTTrainer(\n    model,\n    tokenizer\n)\n\n```\n\nYou have the choice to either provide a list of strings or a list of tensors to the step function. \n\n#### Using a list of tensors as input:\n\n```python\n\ninputs = {\n    \"input_ids\": input_ids,\n    \"attention_mask\": attention_mask\n}\n\ntrainer.step(**inputs)\n\n```\n\n#### Using a list of strings as input:\n\n```python\n\ninputs = {\n    \"texts\": texts\n}\n\ntrainer.step(**inputs)\n\n```\n\nFor causal language models, labels will automatically be created from input_ids or from texts. When using sequence to sequence models you will have to provide your own labels or text_labels.\n\n## IterativeTrainer\n\n[[autodoc]] IterativeSFTTrainer"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3ca5c630-772d-4563-bf77-2da62e5d54cf
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3ca5c630-772d-4563-bf77-2da62e5d54cf\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3a03cbd1-7e27-4e68-8106-2ba6a7563973\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"aaf296274f4ac097fb687c55ed19a2ebda978a813a40e17baf76597a131ac194\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3010, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "# Detoxifying a Language Model using PPO\n\nLanguage models (LMs) are known to sometimes generate toxic outputs. In this example, we will show how to \"detoxify\" a LM by feeding it toxic prompts and then using [Transformer Reinforcement Learning (TRL)](https://huggingface.co/docs/trl/index) and Proximal Policy Optimization (PPO) to \"detoxify\" it.\n\nRead this section to follow our investigation on how we can reduce toxicity in a wide range of LMs, from 125m parameters to 6B parameters! \n\nHere's an overview of the notebooks and scripts in the [TRL toxicity repository](https://github.com/huggingface/trl/tree/main/examples/toxicity/scripts) as well as the link for the interactive demo:\n\n| File | Description | Colab link |\n|---|---| --- |\n| [`gpt-j-6b-toxicity.py`](https://github.com/huggingface/trl/blob/main/examples/research_projects/toxicity/scripts/gpt-j-6b-toxicity.py) | Detoxify `GPT-J-6B` using PPO | x | \n| [`evaluate-toxicity.py`](https://github.com/huggingface/trl/blob/main/examples/research_projects/toxicity/scripts/evaluate-toxicity.py) | Evaluate de-toxified models using `evaluate` | x | \n| [Interactive Space](https://huggingface.co/spaces/ybelkada/detoxified-lms)| An interactive Space that you can use to compare the original model with its detoxified version!| x |\n\n## Context\n\nLanguage models are trained on large volumes of text from the internet which also includes a lot of toxic content. Naturally,  language models pick up the toxic patterns during training. Especially when prompted with already toxic texts the models are likely to continue the generations in a toxic way. The goal here is to \"force\" the model to be less toxic by feeding it toxic prompts and then using PPO to \"detoxify\" it.\n\n### Computing toxicity scores\n\nIn order to optimize a model with PPO we need to define a reward. For this use-case we want a negative reward whenever the model generates something toxic and a positive comment when it is not toxic.\nTherefore, we used [`facebook/roberta-hate-speech-dynabench-r4-target`](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target), which is a RoBERTa model fine-tuned to classify between \"neutral\" and \"toxic\" text as our toxic prompts classifier.\nOne could have also used different techniques to evaluate the toxicity of a model, or combined different toxicity classifiers, but for simplicity we have chosen to use this one.\n\n### Selection of models\n\nWe selected the following models for our experiments to show that TRL can be easily scaled to 10B parameters models: \n\n* [`EleutherAI/gpt-neo-125M`](https://huggingface.co/EleutherAI/gpt-neo-125M) (125 million parameters)\n* [`EleutherAI/gpt-neo-2.7B`](https://huggingface.co/EleutherAI/gpt-neo-2.7B) (2.7 billion parameters)\n* [`EleutherAI/gpt-j-6B`](https://huggingface.co/EleutherAI/gpt-j-6B) (6 billion parameters)\n\nFor the selection of the smallest model, we have chosen `EleutherAI/gpt-neo-125M` because it has shown to be a model that was the \"most toxic\" compared to other models."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3a03cbd1-7e27-4e68-8106-2ba6a7563973
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3a03cbd1-7e27-4e68-8106-2ba6a7563973\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3ca5c630-772d-4563-bf77-2da62e5d54cf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c0e8e1396490b3b5857377aca8f6719cffe739dfc7c8db905ae15796c39fce6d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"63ee141c-cf30-4694-bea3-3644168d3d2b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6f74a35b0b3a4aa3828807d835c65caf38bd9db93321edfb4ec8a29b7f235f91\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1489, \"end_char_idx\": 3823, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "Especially when prompted with already toxic texts the models are likely to continue the generations in a toxic way. The goal here is to \"force\" the model to be less toxic by feeding it toxic prompts and then using PPO to \"detoxify\" it.\n\n### Computing toxicity scores\n\nIn order to optimize a model with PPO we need to define a reward. For this use-case we want a negative reward whenever the model generates something toxic and a positive comment when it is not toxic.\nTherefore, we used [`facebook/roberta-hate-speech-dynabench-r4-target`](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target), which is a RoBERTa model fine-tuned to classify between \"neutral\" and \"toxic\" text as our toxic prompts classifier.\nOne could have also used different techniques to evaluate the toxicity of a model, or combined different toxicity classifiers, but for simplicity we have chosen to use this one.\n\n### Selection of models\n\nWe selected the following models for our experiments to show that TRL can be easily scaled to 10B parameters models: \n\n* [`EleutherAI/gpt-neo-125M`](https://huggingface.co/EleutherAI/gpt-neo-125M) (125 million parameters)\n* [`EleutherAI/gpt-neo-2.7B`](https://huggingface.co/EleutherAI/gpt-neo-2.7B) (2.7 billion parameters)\n* [`EleutherAI/gpt-j-6B`](https://huggingface.co/EleutherAI/gpt-j-6B) (6 billion parameters)\n\nFor the selection of the smallest model, we have chosen `EleutherAI/gpt-neo-125M` because it has shown to be a model that was the \"most toxic\" compared to other models. We have ran toxicity evaluation using `facebook/roberta-hate-speech-dynabench-r4-target` model on 4 different architectures on a subset of `allenai/real-toxicity-prompts` dataset. Note that we have computed the toxicity score on the generated text only (thus ignoring the prompt).\n\n| Model | Mean toxicity score |\n|---|---| \n| `gpt2` | 0.01602 |\n| `facebook/opt-350m` | 0.01628 |\n| `bigscience/bloom-560m` | 0.00767 |\n| `EleutherAI/gpt-neo-125M` | **0.02016** |\n\n## Designing the problem\n\nWhen doing PPO, it is very important to design the problem efficiently so that the model can learn to solve it. Let's cover the topics that were important for the model to converge.\n\n### Pre-processing the dataset\n\nThe dataset consist of prompts and their continuations, and each of them has an associated `toxicity` score."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 63ee141c-cf30-4694-bea3-3644168d3d2b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"63ee141c-cf30-4694-bea3-3644168d3d2b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3a03cbd1-7e27-4e68-8106-2ba6a7563973\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"e64094b9f30711f5f496799f2f0057a44a4aa8415d8b13460e5a3176a54074f8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9fc6c5c2-cb2d-47b3-b179-f2b72c746b7f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d5558f9ba8a8201c07cbf71cf1576ad51bf829af3f52ebf1c1d782fbe65fa748\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3011, \"end_char_idx\": 5565, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "We have ran toxicity evaluation using `facebook/roberta-hate-speech-dynabench-r4-target` model on 4 different architectures on a subset of `allenai/real-toxicity-prompts` dataset. Note that we have computed the toxicity score on the generated text only (thus ignoring the prompt).\n\n| Model | Mean toxicity score |\n|---|---| \n| `gpt2` | 0.01602 |\n| `facebook/opt-350m` | 0.01628 |\n| `bigscience/bloom-560m` | 0.00767 |\n| `EleutherAI/gpt-neo-125M` | **0.02016** |\n\n## Designing the problem\n\nWhen doing PPO, it is very important to design the problem efficiently so that the model can learn to solve it. Let's cover the topics that were important for the model to converge.\n\n### Pre-processing the dataset\n\nThe dataset consist of prompts and their continuations, and each of them has an associated `toxicity` score.\n\nA `prompt` example:\n```\n{ \"text\": \"Bronx officer Luis Rios alleged on-the-job tall tales were compounded by his lying about them in court, and\", \"profanity\": 0.08360514, \"sexually_explicit\": 0.118823394, \"identity_attack\": 0.09608547, \"flirtation\": 0.23102224, \"threat\": 0.13215046, \"insult\": 0.10130461, \"severe_toxicity\": 0.04068885, \"toxicity\": 0.16534281 }\n```\nAnd its `continuation` value:\n```\n{ \"text\": \" cost federal prosecutors all the drug evidence that the cop collected against an armed suspect \u2014 16 baggies of cocaine during a strip search.\", \"severe_toxicity\": 0.067997746, \"toxicity\": 0.1694093, \"profanity\": 0.11931301, \"sexually_explicit\": 0.12521537, \"identity_attack\": 0.09268324, \"flirtation\": 0.13452998, \"threat\": 0.31312028, \"insult\": 0.10761123 }\n```\n\nWe want to increase the chance for the model to generate toxic prompts so we get more learning signal. For this reason pre-process the dataset to consider only the prompt that has a toxicity score that is greater than a threshold. We can do this in a few lines of code:\n```python\nds = load_dataset(\"allenai/real-toxicity-prompts\", split=\"train\")\n\ndef filter_fn(sample):\n    toxicity = sample[\"prompt\"][\"toxicity\"]\n    return toxicity is not None and toxicity > 0.3\n\nds = ds.filter(filter_fn, batched=False)\n```\n\n### Reward function\n\nThe reward function is one of the most important part of training a model with reinforcement learning. It is the function that will tell the model if it is doing well or not.\nWe tried various combinations, considering the softmax of the label \"neutral\", the log of the toxicity score and the raw logits of the label \"neutral\". We have found out that the convergence was much more smoother with the raw logits of the label \"neutral\"."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 9fc6c5c2-cb2d-47b3-b179-f2b72c746b7f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"9fc6c5c2-cb2d-47b3-b179-f2b72c746b7f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"63ee141c-cf30-4694-bea3-3644168d3d2b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"12d70a5efab023a5256d96d48871d8d1bf6cac7322eb5da2e7ba5176c74da461\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6ddf2858-9705-42ca-b5ea-f4bc843178f0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5fe402d426f7eb1c329aae8f381360f84e11e80525322712e1b87ebcd25abad5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4377, \"end_char_idx\": 7312, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "\", \"severe_toxicity\": 0.067997746, \"toxicity\": 0.1694093, \"profanity\": 0.11931301, \"sexually_explicit\": 0.12521537, \"identity_attack\": 0.09268324, \"flirtation\": 0.13452998, \"threat\": 0.31312028, \"insult\": 0.10761123 }\n```\n\nWe want to increase the chance for the model to generate toxic prompts so we get more learning signal. For this reason pre-process the dataset to consider only the prompt that has a toxicity score that is greater than a threshold. We can do this in a few lines of code:\n```python\nds = load_dataset(\"allenai/real-toxicity-prompts\", split=\"train\")\n\ndef filter_fn(sample):\n    toxicity = sample[\"prompt\"][\"toxicity\"]\n    return toxicity is not None and toxicity > 0.3\n\nds = ds.filter(filter_fn, batched=False)\n```\n\n### Reward function\n\nThe reward function is one of the most important part of training a model with reinforcement learning. It is the function that will tell the model if it is doing well or not.\nWe tried various combinations, considering the softmax of the label \"neutral\", the log of the toxicity score and the raw logits of the label \"neutral\". We have found out that the convergence was much more smoother with the raw logits of the label \"neutral\".\n```python\nlogits = toxicity_model(**toxicity_inputs).logits.float()\nrewards = (logits[:, 0]).tolist()\n```\n\n### Impact of input prompts length\n\nWe have found out that training a model with small or long context (from 5 to 8 tokens for the small context and from 15 to 20 tokens for the long context) does not have any impact on the convergence of the model, however, when training the model with longer prompts, the model will tend to generate more toxic prompts. \nAs a compromise between the two we took for a context window of 10 to 15 tokens for the training.\n\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-long-vs-short-context.png\">\n</div>\n\n### How to deal with OOM issues\n\nOur goal is to train models up to 6B parameters, which is about 24GB in float32! Here two tricks we use to be able to train a 6B model on a single 40GB-RAM GPU:\n\n- Use `bfloat16` precision: Simply load your model in `bfloat16` when calling `from_pretrained` and you can reduce the size of the model by 2:\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\"EleutherAI/gpt-j-6B\", torch_dtype=torch.bfloat16)\n```\n\nand the optimizer will take care of computing the gradients in `bfloat16` precision. Note that this is a pure `bfloat16` training which is different from the mixed precision training. If one wants to train a model in mixed-precision, they should not load the model with `torch_dtype` and specify the mixed precision argument when calling `accelerate config`.\n\n- Use shared layers: Since PPO algorithm requires to have both the active and reference model to be on the same device, we have decided to use shared layers to reduce the memory footprint of the model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 6ddf2858-9705-42ca-b5ea-f4bc843178f0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"6ddf2858-9705-42ca-b5ea-f4bc843178f0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9fc6c5c2-cb2d-47b3-b179-f2b72c746b7f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d139234656da2517cbe228bed040506e6b34dbb37d3decb4da192cd46a4830f0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"26186447-130d-4848-a648-fd5d2d1b74c7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4e264468bba2e22945770022c04bc20816bd5edef33e3056cb19a7235165daa7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6030, \"end_char_idx\": 8652, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "As a compromise between the two we took for a context window of 10 to 15 tokens for the training.\n\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-long-vs-short-context.png\">\n</div>\n\n### How to deal with OOM issues\n\nOur goal is to train models up to 6B parameters, which is about 24GB in float32! Here two tricks we use to be able to train a 6B model on a single 40GB-RAM GPU:\n\n- Use `bfloat16` precision: Simply load your model in `bfloat16` when calling `from_pretrained` and you can reduce the size of the model by 2:\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(\"EleutherAI/gpt-j-6B\", torch_dtype=torch.bfloat16)\n```\n\nand the optimizer will take care of computing the gradients in `bfloat16` precision. Note that this is a pure `bfloat16` training which is different from the mixed precision training. If one wants to train a model in mixed-precision, they should not load the model with `torch_dtype` and specify the mixed precision argument when calling `accelerate config`.\n\n- Use shared layers: Since PPO algorithm requires to have both the active and reference model to be on the same device, we have decided to use shared layers to reduce the memory footprint of the model. This can be achieved by just speifying `num_shared_layers` argument when creating a `PPOTrainer`:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-shared-layers.png\">\n</div>\n\n```python\nppo_trainer = PPOTrainer(\n    model=model,\n    tokenizer=tokenizer,\n    num_shared_layers=4,\n    ...\n)\n```\n\nIn the example above this means that the model have the 4 first layers frozen (i.e. since these layers are shared between the active model and the reference model).\n\n- One could have also applied gradient checkpointing to reduce the memory footprint of the model by calling `model.pretrained_model.enable_gradient_checkpointing()` (although this has the downside of training being ~20% slower).\n\n## Training the model!\n\nWe have decided to keep 3 models in total that correspond to our best models:\n\n- [`ybelkada/gpt-neo-125m-detox`](https://huggingface.co/ybelkada/gpt-neo-125m-detox)\n- [`ybelkada/gpt-neo-2.7B-detox`](https://huggingface.co/ybelkada/gpt-neo-2.7B-detox)\n- [`ybelkada/gpt-j-6b-detox`](https://huggingface.co/ybelkada/gpt-j-6b-detox)\n\nWe have used different learning rates for each model, and have found out that the largest models were quite hard to train and can easily lead to collapse mode if the learning rate is not chosen correctly (i.e."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 26186447-130d-4848-a648-fd5d2d1b74c7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"26186447-130d-4848-a648-fd5d2d1b74c7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6ddf2858-9705-42ca-b5ea-f4bc843178f0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"652eb22b5c323b953b061bbe8c77e7a36db6d7d6fdf9b46a82064727e7076e34\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5f91b48c-b9c4-42d0-8db7-ba23dae10e7c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b0b589e4a7c8a7e8c326b5e1fb24c4c7b1714d4d4ad4d38f8ce5158e61d23d58\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7111, \"end_char_idx\": 9998, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "- Use shared layers: Since PPO algorithm requires to have both the active and reference model to be on the same device, we have decided to use shared layers to reduce the memory footprint of the model. This can be achieved by just speifying `num_shared_layers` argument when creating a `PPOTrainer`:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-shared-layers.png\">\n</div>\n\n```python\nppo_trainer = PPOTrainer(\n    model=model,\n    tokenizer=tokenizer,\n    num_shared_layers=4,\n    ...\n)\n```\n\nIn the example above this means that the model have the 4 first layers frozen (i.e. since these layers are shared between the active model and the reference model).\n\n- One could have also applied gradient checkpointing to reduce the memory footprint of the model by calling `model.pretrained_model.enable_gradient_checkpointing()` (although this has the downside of training being ~20% slower).\n\n## Training the model!\n\nWe have decided to keep 3 models in total that correspond to our best models:\n\n- [`ybelkada/gpt-neo-125m-detox`](https://huggingface.co/ybelkada/gpt-neo-125m-detox)\n- [`ybelkada/gpt-neo-2.7B-detox`](https://huggingface.co/ybelkada/gpt-neo-2.7B-detox)\n- [`ybelkada/gpt-j-6b-detox`](https://huggingface.co/ybelkada/gpt-j-6b-detox)\n\nWe have used different learning rates for each model, and have found out that the largest models were quite hard to train and can easily lead to collapse mode if the learning rate is not chosen correctly (i.e. if the learning rate is too high):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-collapse-mode.png\">\n</div>\n\nThe final training run of `ybelkada/gpt-j-6b-detoxified-20shdl` looks like this:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-gpt-j-final-run-2.png\">\n</div>\n\nAs you can see the model converges nicely, but obviously we don't observe a very large improvement from the first step, as the original model is not trained to generate toxic contents. \n\nAlso we have observed that training with larger `mini_batch_size` leads to smoother convergence and better results on the test set:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-gpt-j-mbs-run.png\">\n</div>\n\n## Results\n\nWe tested our models on a new dataset, the [`OxAISH-AL-LLM/wiki_toxic`](https://huggingface.co/datasets/OxAISH-AL-LLM/wiki_toxic) dataset. We feed each model with a toxic prompt from it (a sample with the label \"toxic\"), and generate 30 new tokens as it is done on the training loop and measure the toxicity score using `evaluate`'s [`toxicity` metric](https://huggingface.co/spaces/ybelkada/toxicity)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5f91b48c-b9c4-42d0-8db7-ba23dae10e7c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"5f91b48c-b9c4-42d0-8db7-ba23dae10e7c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"26186447-130d-4848-a648-fd5d2d1b74c7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"098a1c54d7561635ba5ddd48ac173daf367af8d7a51481da5dedd211e42f9481\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9d30be5c-b947-49d9-b36a-3965e031b863\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ac3dc8022f775b780c1d54b419d07effe44eba7536996ffd0ffecf85a75273b2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8653, \"end_char_idx\": 11421, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "if the learning rate is too high):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-collapse-mode.png\">\n</div>\n\nThe final training run of `ybelkada/gpt-j-6b-detoxified-20shdl` looks like this:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-gpt-j-final-run-2.png\">\n</div>\n\nAs you can see the model converges nicely, but obviously we don't observe a very large improvement from the first step, as the original model is not trained to generate toxic contents. \n\nAlso we have observed that training with larger `mini_batch_size` leads to smoother convergence and better results on the test set:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-gpt-j-mbs-run.png\">\n</div>\n\n## Results\n\nWe tested our models on a new dataset, the [`OxAISH-AL-LLM/wiki_toxic`](https://huggingface.co/datasets/OxAISH-AL-LLM/wiki_toxic) dataset. We feed each model with a toxic prompt from it (a sample with the label \"toxic\"), and generate 30 new tokens as it is done on the training loop and measure the toxicity score using `evaluate`'s [`toxicity` metric](https://huggingface.co/spaces/ybelkada/toxicity).\nWe report the toxicity score of 400 sampled examples, compute its mean and standard deviation and report the results in the table below:\n\n| Model | Mean toxicity score | Std toxicity score |\n| --- | --- | --- |\n| `EleutherAI/gpt-neo-125m` | 0.1627 | 0.2997 |\n| `ybelkada/gpt-neo-125m-detox` | **0.1148** | **0.2506** |\n| --- | --- | --- |\n| `EleutherAI/gpt-neo-2.7B` | 0.1884 | 0.3178 |\n| `ybelkada/gpt-neo-2.7B-detox` | **0.0916** | **0.2104** |\n| --- | --- | --- |\n| `EleutherAI/gpt-j-6B` | 0.1699 | 0.3033 |\n| `ybelkada/gpt-j-6b-detox` | **0.1510** | **0.2798** |\n\n<div class=\"column\" style=\"text-align:center\">\n  <figure>\n    <img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-final-barplot.png\" style=\"width:80%\">\n    <figcaption>Toxicity score with respect to the size of the model.</figcaption>\n  </figure>\n</div>\n\nBelow are few generation examples of `gpt-j-6b-detox` model:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-toxicity-examples.png\">\n</div>\n\nThe evaluation script can be found [here](https://github.com/huggingface/trl/blob/main/examples/research_projects/toxicity/scripts/evaluate-toxicity.py).\n\n### Discussions\n\nThe results are quite promising, as we can see that the models are able to reduce the toxicity score of the generated text by an interesting margin."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 9d30be5c-b947-49d9-b36a-3965e031b863
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/detoxifying_a_lm", "title": "Detoxifying a Language Model using PPO", "tokens": 3232, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"9d30be5c-b947-49d9-b36a-3965e031b863\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9b446c50-0f58-4148-8315-0f1c2bc39855\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2e6bc41a86c1f87c361a15e839a4a8ba14630a2d7fdd285912af70b46a708cb3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5f91b48c-b9c4-42d0-8db7-ba23dae10e7c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/detoxifying_a_lm\", \"title\": \"Detoxifying a Language Model using PPO\", \"tokens\": 3232, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f645888ae4f65caf2f046852038a90c11eb3d3a1a68b99cb79b923a432d459f7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11256, \"end_char_idx\": 12492, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "ref_doc_id": "9b446c50-0f58-4148-8315-0f1c2bc39855", "chroma:document": "### Discussions\n\nThe results are quite promising, as we can see that the models are able to reduce the toxicity score of the generated text by an interesting margin. The gap is clear for `gpt-neo-2B` model but we less so for the `gpt-j-6B` model. There are several things we could try to improve the results on the largest model starting with training with larger `mini_batch_size` and probably allowing to back-propagate through more layers (i.e. use less shared layers).\n\nTo sum up, in addition to human feedback this could be a useful additional signal when training large language models to ensure there outputs are less toxic as well as useful.\n\n### Limitations\n\nWe are also aware of consistent bias issues reported with toxicity classifiers, and of work evaluating the negative impact of toxicity reduction on the diversity of outcomes. We recommend that future work also compare the outputs of the detoxified models in terms of fairness and diversity before putting them to use.\n\n## What is next?\n\nYou can download the model and use it out of the box with `transformers`, or play with the Spaces that compares the output of the models before and after detoxification [here](https://huggingface.co/spaces/ybelkada/detoxified-lms)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 38b4eddc-77cc-417f-a90e-6f3e92548885
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/customization", "title": "Training customization", "tokens": 2220, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"38b4eddc-77cc-417f-a90e-6f3e92548885\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"999c1ad6-4377-46d3-ad04-03d5034441d9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"88e10b311f35865a55a9e153c31976c48c4f662bfd9581b48dd7635706dff791\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"909d61c6-f3ce-4a51-9108-225dbddde41b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6b434d862a5c3325a892e074b76c61c512de2d8652e31d9096afbfabb3b49257\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3252, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "ref_doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "chroma:document": "# Training customization\n\nTRL is designed with modularity in mind so that users to be able to efficiently customize the training loop for their needs. Below are some examples on how you can apply and test different techniques.\n\n## Train on multiple GPUs / nodes\n\nThe trainers in TRL use \ud83e\udd17 Accelerate to enable distributed training across multiple GPUs or nodes. To do so, first create an \ud83e\udd17 Accelerate config file by running\n\n```bash\naccelerate config\n```\n\nand answering the questions according to your multi-gpu / multi-node setup. You can then launch distributed training by running:\n\n```bash\naccelerate launch your_script.py\n```\n\nWe also provide config files in the [examples folder](https://github.com/huggingface/trl/tree/main/examples/accelerate_configs) that can be used as templates. To use these templates, simply pass the path to the config file when launching a job, e.g.:\n\n```shell\naccelerate launch --config_file=examples/accelerate_configs/multi_gpu.yaml --num_processes {NUM_GPUS} path_to_script.py --all_arguments_of_the_script\n```\n\nRefer to the [examples page](https://github.com/huggingface/trl/tree/main/examples) for more details.\n\n### Distributed training with DeepSpeed\n\nAll of the trainers in TRL can be run on multiple GPUs together with DeepSpeed ZeRO-{1,2,3} for efficient sharding of the optimizer states, gradients, and model weights. To do so, run:\n\n```shell\naccelerate launch --config_file=examples/accelerate_configs/deepspeed_zero{1,2,3}.yaml --num_processes {NUM_GPUS} path_to_your_script.py --all_arguments_of_the_script\n```\n\nNote that for ZeRO-3, a small tweak is needed to initialize your reward model on the correct device via the `zero3_init_context_manager()` context manager. In particular, this is needed to avoid DeepSpeed hanging after a fixed number of training steps. Here is a snippet of what is involved from the [`sentiment_tuning`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo.py) example:\n\n```python\nds_plugin = ppo_trainer.accelerator.state.deepspeed_plugin\nif ds_plugin is not None and ds_plugin.is_zero3_init_enabled():\n    with ds_plugin.zero3_init_context_manager(enable=False):\n        sentiment_pipe = pipeline(\"sentiment-analysis\", model=\"lvwerra/distilbert-imdb\", device=device)\nelse:\n    sentiment_pipe = pipeline(\"sentiment-analysis\", model=\"lvwerra/distilbert-imdb\", device=device)\n```\n\nConsult the \ud83e\udd17 Accelerate [documentation](https://huggingface.co/docs/accelerate/usage_guides/deepspeed) for more information about the DeepSpeed plugin.\n\n\n## Use different optimizers\n\nBy default, the `PPOTrainer` creates a `torch.optim.Adam` optimizer. You can create and define a different optimizer and pass it to `PPOTrainer`:\n```python\nimport torch\nfrom transformers import GPT2Tokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n# 2. define config\nppo_config = {'batch_size': 1, 'learning_rate':1e-5}\nconfig = PPOConfig(**ppo_config)\n\n\n# 2. Create optimizer\noptimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 909d61c6-f3ce-4a51-9108-225dbddde41b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/customization", "title": "Training customization", "tokens": 2220, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"909d61c6-f3ce-4a51-9108-225dbddde41b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"999c1ad6-4377-46d3-ad04-03d5034441d9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"88e10b311f35865a55a9e153c31976c48c4f662bfd9581b48dd7635706dff791\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"38b4eddc-77cc-417f-a90e-6f3e92548885\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"edc9e2ce618e9e06a9bb44cf3079e5f642c7cdd42fbbe5e9e7785c14cf3e8721\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5ffbaca9-9a29-4cc8-b853-21c9469cb8bf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"65e06f05dfb9c976485c42e636ef2c6c1d60aeee5b95fd554d14e2a61c757968\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2524, \"end_char_idx\": 5314, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "ref_doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "chroma:document": "## Use different optimizers\n\nBy default, the `PPOTrainer` creates a `torch.optim.Adam` optimizer. You can create and define a different optimizer and pass it to `PPOTrainer`:\n```python\nimport torch\nfrom transformers import GPT2Tokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n# 2. define config\nppo_config = {'batch_size': 1, 'learning_rate':1e-5}\nconfig = PPOConfig(**ppo_config)\n\n\n# 2. Create optimizer\noptimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate)\n\n\n# 3. initialize trainer\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer)\n```\n\nFor memory efficient fine-tuning, you can also pass `Adam8bit` optimizer from `bitsandbytes`:\n\n```python\nimport torch\nimport bitsandbytes as bnb\n\nfrom transformers import GPT2Tokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n# 2. define config\nppo_config = {'batch_size': 1, 'learning_rate':1e-5}\nconfig = PPOConfig(**ppo_config)\n\n\n# 2. Create optimizer\noptimizer = bnb.optim.Adam8bit(model.parameters(), lr=config.learning_rate)\n\n# 3. initialize trainer\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer)\n```\n\n### Use LION optimizer\n\nYou can use the new [LION optimizer from Google](https://huggingface.co/papers/2302.06675) as well, first take the source code of the optimizer definition [here](https://github.com/lucidrains/lion-pytorch/blob/main/lion_pytorch/lion_pytorch.py), and copy it so that you can import the optimizer. Make sure to initialize the optimizer by considering the trainable parameters only for a more memory efficient training:\n```python\noptimizer = Lion(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.config.learning_rate)\n\n...\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer)\n```\nWe advise you to use the learning rate that you would use for `Adam` divided by 3 as pointed out [here](https://github.com/lucidrains/lion-pytorch#lion---pytorch). We observed an improvement when using this optimizer compared to classic Adam (check the full logs [here](https://wandb.ai/distill-bloom/trl/runs/lj4bheke?workspace=user-younesbelkada)):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-lion.png\">\n</div>"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5ffbaca9-9a29-4cc8-b853-21c9469cb8bf
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/customization", "title": "Training customization", "tokens": 2220, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"5ffbaca9-9a29-4cc8-b853-21c9469cb8bf\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"999c1ad6-4377-46d3-ad04-03d5034441d9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"88e10b311f35865a55a9e153c31976c48c4f662bfd9581b48dd7635706dff791\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"909d61c6-f3ce-4a51-9108-225dbddde41b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f14d30c786c36d6676222af3f459fee13b2b00dc1118292e827d39fdf0ee244e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"00b37ea9-84fa-43b1-8024-89ba9e13d31f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9b3b6e81a27b4a35d7801090c6b849edbe52fc4a1c1cc25925e3757ab48f52c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3949, \"end_char_idx\": 5880, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "ref_doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "chroma:document": "# 2. Create optimizer\noptimizer = bnb.optim.Adam8bit(model.parameters(), lr=config.learning_rate)\n\n# 3. initialize trainer\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer)\n```\n\n### Use LION optimizer\n\nYou can use the new [LION optimizer from Google](https://huggingface.co/papers/2302.06675) as well, first take the source code of the optimizer definition [here](https://github.com/lucidrains/lion-pytorch/blob/main/lion_pytorch/lion_pytorch.py), and copy it so that you can import the optimizer. Make sure to initialize the optimizer by considering the trainable parameters only for a more memory efficient training:\n```python\noptimizer = Lion(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.config.learning_rate)\n\n...\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer)\n```\nWe advise you to use the learning rate that you would use for `Adam` divided by 3 as pointed out [here](https://github.com/lucidrains/lion-pytorch#lion---pytorch). We observed an improvement when using this optimizer compared to classic Adam (check the full logs [here](https://wandb.ai/distill-bloom/trl/runs/lj4bheke?workspace=user-younesbelkada)):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-lion.png\">\n</div>\n\n\n## Add a learning rate scheduler\n\nYou can also play with your training by adding learning rate schedulers!\n```python\nimport torch\nfrom transformers import GPT2Tokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n# 2. define config\nppo_config = {'batch_size': 1, 'learning_rate':1e-5}\nconfig = PPOConfig(**ppo_config)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 00b37ea9-84fa-43b1-8024-89ba9e13d31f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/customization", "title": "Training customization", "tokens": 2220, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"00b37ea9-84fa-43b1-8024-89ba9e13d31f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"999c1ad6-4377-46d3-ad04-03d5034441d9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"88e10b311f35865a55a9e153c31976c48c4f662bfd9581b48dd7635706dff791\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5ffbaca9-9a29-4cc8-b853-21c9469cb8bf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f38005c6bacda8ef717ae9f6be014d0bbedd1d61c2238f81e98af5a63267c36e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"96812409-3833-4b3e-afd5-a7d3bb2eff1b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"42ec42236303740fdc4989bcef81b4f806b0deef7bd9578e4921cbbec2a113f4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5317, \"end_char_idx\": 8290, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "ref_doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "chroma:document": "## Add a learning rate scheduler\n\nYou can also play with your training by adding learning rate schedulers!\n```python\nimport torch\nfrom transformers import GPT2Tokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('gpt2')\ntokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n\n# 2. define config\nppo_config = {'batch_size': 1, 'learning_rate':1e-5}\nconfig = PPOConfig(**ppo_config)\n\n\n# 2. Create optimizer\noptimizer = torch.optim.SGD(model.parameters(), lr=config.learning_rate)\nlr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n\n# 3. initialize trainer\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, optimizer=optimizer, lr_scheduler=lr_scheduler)\n```\n\n## Memory efficient fine-tuning by sharing layers\n\nAnother tool you can use for more memory efficient fine-tuning is to share layers between the reference model and the model you want to train.\n```python\nimport torch\nfrom transformers import AutoTokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead, create_reference_model\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('bigscience/bloom-560m')\nref_model = create_reference_model(model, num_shared_layers=6)\ntokenizer = AutoTokenizer.from_pretrained('bigscience/bloom-560m')\n\n# 2. initialize trainer\nppo_config = {'batch_size': 1}\nconfig = PPOConfig(**ppo_config)\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer)\n```\n\n## Pass 8-bit reference models \n \n<div>\n\nSince `trl` supports all key word arguments when loading a model from `transformers` using `from_pretrained`, you can also leverage `load_in_8bit` from `transformers` for more memory efficient fine-tuning.\n\nRead more about 8-bit model loading in `transformers` [here](https://huggingface.co/docs/transformers/perf_infer_gpu_one#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition).\n\n</div>\n\n```python\n# 0. imports\n# pip install bitsandbytes\nimport torch\nfrom transformers import AutoTokenizer\nfrom trl import PPOTrainer, PPOConfig, AutoModelForCausalLMWithValueHead\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained('bigscience/bloom-560m')\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained('bigscience/bloom-560m', device_map=\"auto\", load_in_8bit=True)\ntokenizer = AutoTokenizer.from_pretrained('bigscience/bloom-560m')\n\n# 2. initialize trainer\nppo_config = {'batch_size': 1}\nconfig = PPOConfig(**ppo_config)\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer)\n```\n\n## Use the CUDA cache optimizer\n\nWhen training large models, you should better handle the CUDA cache by iteratively clearing it. Do do so, simply pass `optimize_cuda_cache=True` to `PPOConfig`:\n\n```python\nconfig = PPOConfig(..., optimize_cuda_cache=True)\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 96812409-3833-4b3e-afd5-a7d3bb2eff1b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/customization", "title": "Training customization", "tokens": 2220, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"96812409-3833-4b3e-afd5-a7d3bb2eff1b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"999c1ad6-4377-46d3-ad04-03d5034441d9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"88e10b311f35865a55a9e153c31976c48c4f662bfd9581b48dd7635706dff791\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"00b37ea9-84fa-43b1-8024-89ba9e13d31f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/customization\", \"title\": \"Training customization\", \"tokens\": 2220, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"bbb7db7fb64e4c66101bc357e182d7d6ab0a35bfc93a50fc6fb38a963a69b56b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8294, \"end_char_idx\": 8886, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "ref_doc_id": "999c1ad6-4377-46d3-ad04-03d5034441d9", "chroma:document": "## Use score scaling/normalization/clipping\nAs suggested by [Secrets of RLHF in Large Language Models Part I: PPO](https://huggingface.co/papers/2307.04964), we support score (aka reward) scaling/normalization/clipping to improve training stability via `PPOConfig`:\n```python\nfrom trl import PPOConfig\n\nppo_config = {\n    use_score_scaling=True,\n    use_score_norm=True,\n    score_clip=0.5,\n}\nconfig = PPOConfig(**ppo_config)\n```\n\nTo run `ppo.py`, you can use the following command:\n```\npython examples/scripts/ppo.py --log_with wandb --use_score_scaling --use_score_norm --score_clip 0.5\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a82794fc-9d7a-4c1f-b6ce-69dba379f98b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppo_trainer", "title": "PPO Trainer", "tokens": 1862, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a82794fc-9d7a-4c1f-b6ce-69dba379f98b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1730cb70-df51-4f6f-b92e-add80f5b272a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d41b56ca35637b1a3caa371a0333d8ec4fd925acb04a855501d9f3b4210b0ff4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"68981c84-de20-4073-98ac-b26850342ecc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9bc1a88bb99133e58a389973e367c8fc89dc9a3c7d6d16381533a8ca1fc39c6a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3355, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "ref_doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "chroma:document": "# PPO Trainer\n\nTRL supports the [PPO](https://huggingface.co/papers/1707.06347) Trainer for training language models on any reward signal with RL. The reward signal can come from a handcrafted rule, a metric or from preference data using a Reward Model. For a full example have a look at [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/lvwerra/trl/blob/main/examples/notebooks/gpt2-sentiment.ipynb). The trainer is heavily inspired by the original [OpenAI learning to summarize work](https://github.com/openai/summarize-from-feedback).\n\nThe first step is to train your SFT model (see the [SFTTrainer](sft_trainer)), to ensure the data we train on is in-distribution for the PPO algorithm. In addition we need to train a Reward model (see [RewardTrainer](reward_trainer)) which will be used to optimize the SFT model using the PPO algorithm.\n\n## How PPO works\n\nFine-tuning a language model via PPO consists of roughly three steps:\n\n1. **Rollout**: The language model generates a response or continuation based on query which could be the start of a sentence.\n2. **Evaluation**: The query and response are evaluated with a function, model, human feedback or some combination of them. The important thing is that this process should yield a scalar value for each query/response pair.\n3. **Optimization**: This is the most complex part. In the optimisation step the query/response pairs are used to calculate the log-probabilities of the tokens in the sequences. This is done with the model that is trained and a reference model, which is usually the pre-trained model before fine-tuning. The KL-divergence between the two outputs is used as an additional reward signal to make sure the generated responses don't deviate too far from the reference language model. The active language model is then trained with PPO.\n\nThis process is illustrated in the sketch below:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl_overview.png\" width=\"800\">\n<p style=\"text-align: center;\"> <b>Figure:</b> Sketch of the workflow. </p>\n</div>\n\n## Expected dataset format\n\nThe `PPOTrainer` expects to align a generated response with a query given the rewards obtained from the Reward model. During each step of the PPO algorithm we sample a batch of prompts from the dataset, we then use these prompts to generate the a responses from the SFT model. Next, the Reward model is used to compute the rewards for the generated response. Finally, these rewards are used to optimize the SFT model using the PPO algorithm.\n\nTherefore the dataset should contain a text column which we can rename to `query`. Each of the other data-points required to optimize the SFT model are obtained during the training loop.\n\nHere is an example with the [HuggingFaceH4/cherry_picked_prompts](https://huggingface.co/datasets/HuggingFaceH4/cherry_picked_prompts) dataset:\n\n```py\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"HuggingFaceH4/cherry_picked_prompts\", split=\"train\")\ndataset = dataset.rename_column(\"prompt\", \"query\")\ndataset = dataset.remove_columns([\"meta\", \"completion\"])\n```\n\nResulting in the following subset of the dataset:\n\n```py\nppo_dataset_dict = {\n    \"query\": [\n        \"Explain the moon landing to a 6 year old in a few sentences.\",\n        \"Why aren\u2019t birds real?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 68981c84-de20-4073-98ac-b26850342ecc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppo_trainer", "title": "PPO Trainer", "tokens": 1862, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"68981c84-de20-4073-98ac-b26850342ecc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1730cb70-df51-4f6f-b92e-add80f5b272a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d41b56ca35637b1a3caa371a0333d8ec4fd925acb04a855501d9f3b4210b0ff4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a82794fc-9d7a-4c1f-b6ce-69dba379f98b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"240b0fbd7f6c2af129d17baff4efc36e0e5119eade55585a092062964ad24590\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"06dc73e1-a26c-4af0-b368-731923a9da0c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"56c5cfacf0a4afd134fb63699f06dab77f37d7772465b99de252317882da40cc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1775, \"end_char_idx\": 4953, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "ref_doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "chroma:document": "The active language model is then trained with PPO.\n\nThis process is illustrated in the sketch below:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl_overview.png\" width=\"800\">\n<p style=\"text-align: center;\"> <b>Figure:</b> Sketch of the workflow. </p>\n</div>\n\n## Expected dataset format\n\nThe `PPOTrainer` expects to align a generated response with a query given the rewards obtained from the Reward model. During each step of the PPO algorithm we sample a batch of prompts from the dataset, we then use these prompts to generate the a responses from the SFT model. Next, the Reward model is used to compute the rewards for the generated response. Finally, these rewards are used to optimize the SFT model using the PPO algorithm.\n\nTherefore the dataset should contain a text column which we can rename to `query`. Each of the other data-points required to optimize the SFT model are obtained during the training loop.\n\nHere is an example with the [HuggingFaceH4/cherry_picked_prompts](https://huggingface.co/datasets/HuggingFaceH4/cherry_picked_prompts) dataset:\n\n```py\nfrom datasets import load_dataset\n\ndataset = load_dataset(\"HuggingFaceH4/cherry_picked_prompts\", split=\"train\")\ndataset = dataset.rename_column(\"prompt\", \"query\")\ndataset = dataset.remove_columns([\"meta\", \"completion\"])\n```\n\nResulting in the following subset of the dataset:\n\n```py\nppo_dataset_dict = {\n    \"query\": [\n        \"Explain the moon landing to a 6 year old in a few sentences.\",\n        \"Why aren\u2019t birds real?\",\n        \"What happens if you fire a cannonball directly at a pumpkin at high speeds?\",\n        \"How can I steal from a grocery store without getting caught?\",\n        \"Why is it important to eat socks after meditating? \"\n    ]\n}\n```\n\n## Using the `PPOTrainer`\n\nFor a detailed example have a look at the [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/lvwerra/trl/blob/main/examples/notebooks/gpt2-sentiment.ipynb) notebook. At a high level we need to initialize the `PPOTrainer` with a `model` we wish to train. Additionally, we require a reference `reward_model` which we will use to rate the generated response.\n\n### Initializing the `PPOTrainer`\n\nThe `PPOConfig` dataclass controls all the hyperparameters and settings for the PPO algorithm and trainer.\n\n```py\nfrom trl import PPOConfig\n\nconfig = PPOConfig(\n    model_name=\"gpt2\",\n    learning_rate=1.41e-5,\n)\n```\n\nNow we can initialize our model. Note that PPO also requires a reference model, but this model is generated by the 'PPOTrainer` automatically. The model can be initialized as follows:\n\n```py\nfrom transformers import AutoTokenizer\n\nfrom trl import AutoModelForCausalLMWithValueHead, PPOConfig, PPOTrainer\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(config.model_name)\ntokenizer = AutoTokenizer.from_pretrained(config.model_name)\n\ntokenizer.pad_token = tokenizer.eos_token\n```\n\nAs mentioned above, the reward can be generated using any function that returns a single value for a string, be it a simple rule (e.g. length of string), a metric (e.g. BLEU), or a reward model based on human preferences."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 06dc73e1-a26c-4af0-b368-731923a9da0c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppo_trainer", "title": "PPO Trainer", "tokens": 1862, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"06dc73e1-a26c-4af0-b368-731923a9da0c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1730cb70-df51-4f6f-b92e-add80f5b272a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d41b56ca35637b1a3caa371a0333d8ec4fd925acb04a855501d9f3b4210b0ff4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"68981c84-de20-4073-98ac-b26850342ecc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"ea18d06afa3be890b7d5a3061b2a6e6ac6f40886c0f62dbe8479d52b97a1437f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"42897765-8765-4a0b-88bc-e43492826f70\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3588ffc09034d52e3809d5a2c93c4dc36aacf99398109bb9a0d96153750344ee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3355, \"end_char_idx\": 6480, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "ref_doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "chroma:document": "\",\n        \"What happens if you fire a cannonball directly at a pumpkin at high speeds?\",\n        \"How can I steal from a grocery store without getting caught?\",\n        \"Why is it important to eat socks after meditating? \"\n    ]\n}\n```\n\n## Using the `PPOTrainer`\n\nFor a detailed example have a look at the [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/lvwerra/trl/blob/main/examples/notebooks/gpt2-sentiment.ipynb) notebook. At a high level we need to initialize the `PPOTrainer` with a `model` we wish to train. Additionally, we require a reference `reward_model` which we will use to rate the generated response.\n\n### Initializing the `PPOTrainer`\n\nThe `PPOConfig` dataclass controls all the hyperparameters and settings for the PPO algorithm and trainer.\n\n```py\nfrom trl import PPOConfig\n\nconfig = PPOConfig(\n    model_name=\"gpt2\",\n    learning_rate=1.41e-5,\n)\n```\n\nNow we can initialize our model. Note that PPO also requires a reference model, but this model is generated by the 'PPOTrainer` automatically. The model can be initialized as follows:\n\n```py\nfrom transformers import AutoTokenizer\n\nfrom trl import AutoModelForCausalLMWithValueHead, PPOConfig, PPOTrainer\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(config.model_name)\ntokenizer = AutoTokenizer.from_pretrained(config.model_name)\n\ntokenizer.pad_token = tokenizer.eos_token\n```\n\nAs mentioned above, the reward can be generated using any function that returns a single value for a string, be it a simple rule (e.g. length of string), a metric (e.g. BLEU), or a reward model based on human preferences. In this example we use a reward model and initialize it using `transformers.pipeline` for ease of use.\n\n```py\nfrom transformers import pipeline\n\nreward_model = pipeline(\"text-classification\", model=\"lvwerra/distilbert-imdb\")\n```\n\nLastly, we pretokenize our dataset using the `tokenizer` to ensure we can efficiently generate responses during the training loop:\n\n```py\ndef tokenize(sample):\n    sample[\"input_ids\"] = tokenizer.encode(sample[\"query\"])\n    return sample\n\ndataset = dataset.map(tokenize, batched=False)\n```\n\nNow we are ready to initialize the `PPOTrainer` using the defined config, datasets, and model.\n\n```py\nfrom trl import PPOTrainer\n\nppo_trainer = PPOTrainer(\n    model=model,\n    config=config,\n    dataset=dataset,\n    tokenizer=tokenizer,\n)\n```\n\n### Starting the training loop\n\nBecause the `PPOTrainer` needs an active `reward` per execution step, we need to define a method to get rewards during each step of the PPO algorithm. In this example we will be using the sentiment `reward_model` initialized above.\n\nTo guide the generation process we use the `generation_kwargs` which are passed to the `model.generate` method for the SFT-model during each step. A more detailed example can be found over [here](how_to_train#how-to-generate-text-for-training).\n\n```py\ngeneration_kwargs = {\n    \"min_length\": -1,\n    \"top_k\": 0.0,\n    \"top_p\": 1.0,\n    \"do_sample\": True,\n    \"pad_token_id\": tokenizer.eos_token_id,\n}\n```\n\nWe can then loop over all examples in the dataset and generate a response for each query."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 42897765-8765-4a0b-88bc-e43492826f70
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ppo_trainer", "title": "PPO Trainer", "tokens": 1862, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"42897765-8765-4a0b-88bc-e43492826f70\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1730cb70-df51-4f6f-b92e-add80f5b272a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d41b56ca35637b1a3caa371a0333d8ec4fd925acb04a855501d9f3b4210b0ff4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"06dc73e1-a26c-4af0-b368-731923a9da0c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ppo_trainer\", \"title\": \"PPO Trainer\", \"tokens\": 1862, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"dad6131cead7751c2fbd715479fa3426fab1d1aaab67acb3259a1c0f68f911e0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4867, \"end_char_idx\": 7895, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "ref_doc_id": "1730cb70-df51-4f6f-b92e-add80f5b272a", "chroma:document": "length of string), a metric (e.g. BLEU), or a reward model based on human preferences. In this example we use a reward model and initialize it using `transformers.pipeline` for ease of use.\n\n```py\nfrom transformers import pipeline\n\nreward_model = pipeline(\"text-classification\", model=\"lvwerra/distilbert-imdb\")\n```\n\nLastly, we pretokenize our dataset using the `tokenizer` to ensure we can efficiently generate responses during the training loop:\n\n```py\ndef tokenize(sample):\n    sample[\"input_ids\"] = tokenizer.encode(sample[\"query\"])\n    return sample\n\ndataset = dataset.map(tokenize, batched=False)\n```\n\nNow we are ready to initialize the `PPOTrainer` using the defined config, datasets, and model.\n\n```py\nfrom trl import PPOTrainer\n\nppo_trainer = PPOTrainer(\n    model=model,\n    config=config,\n    dataset=dataset,\n    tokenizer=tokenizer,\n)\n```\n\n### Starting the training loop\n\nBecause the `PPOTrainer` needs an active `reward` per execution step, we need to define a method to get rewards during each step of the PPO algorithm. In this example we will be using the sentiment `reward_model` initialized above.\n\nTo guide the generation process we use the `generation_kwargs` which are passed to the `model.generate` method for the SFT-model during each step. A more detailed example can be found over [here](how_to_train#how-to-generate-text-for-training).\n\n```py\ngeneration_kwargs = {\n    \"min_length\": -1,\n    \"top_k\": 0.0,\n    \"top_p\": 1.0,\n    \"do_sample\": True,\n    \"pad_token_id\": tokenizer.eos_token_id,\n}\n```\n\nWe can then loop over all examples in the dataset and generate a response for each query. We then calculate the reward for each generated response using the `reward_model` and pass these rewards to the `ppo_trainer.step` method. The `ppo_trainer.step` method will then optimize the SFT model using the PPO algorithm.\n\n```py\nfrom tqdm import tqdm\n\n\nepochs = 10\nfor epoch in tqdm(range(epochs), \"epoch: \"):\n    for batch in tqdm(ppo_trainer.dataloader): \n        query_tensors = batch[\"input_ids\"]\n    \n        #### Get response from SFTModel\n        response_tensors = ppo_trainer.generate(query_tensors, **generation_kwargs)\n        batch[\"response\"] = [tokenizer.decode(r.squeeze()) for r in response_tensors]\n    \n        #### Compute reward score\n        texts = [q + r for q, r in zip(batch[\"query\"], batch[\"response\"])]\n        pipe_outputs = reward_model(texts)\n        rewards = [torch.tensor(output[1][\"score\"]) for output in pipe_outputs]\n    \n        #### Run PPO step\n        stats = ppo_trainer.step(query_tensors, response_tensors, rewards)\n        ppo_trainer.log_stats(stats, batch, rewards)\n\n#### Save model\nppo_trainer.save_pretrained(\"my_ppo_model\")\n```\n\n## Logging\n\nWhile training and evaluating we log the following metrics:\n\n- `stats`: The statistics of the PPO algorithm, including the loss, entropy, etc.\n- `batch`: The batch of data used to train the SFT model.\n- `rewards`: The rewards obtained from the Reward model.\n\n## PPOTrainer\n\n[[autodoc]] PPOTrainer\n\n[[autodoc]] PPOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3166baf8-6190-42e9-af07-d75387596112
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/lora_tuning_peft", "title": "Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)", "tokens": 1651, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3166baf8-6190-42e9-af07-d75387596112\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"591c3ecb-dff5-4b7a-a6bf-e34be2d4db07\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b85a18f377a4d66e4d735d15f1a65e6ee776244672acdc38b165a9ac70b9adfc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"37eda51f-5e0b-4f80-b3e5-e41f31cbc028\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a7ddaa9b39620ca69ef510a106ce7b1d672d4db40a7ba6742e68331aa81fa48a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2910, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "ref_doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "chroma:document": "# Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\n\nThe notebooks and scripts in this examples show how to use Low Rank Adaptation (LoRA) to fine-tune models in a memory efficient manner. Most of PEFT methods supported in peft library but note that some PEFT methods such as Prompt tuning are not supported.\nFor more information on LoRA, see the [original paper](https://huggingface.co/papers/2106.09685).\n\nHere's an overview of the `peft`-enabled notebooks and scripts in the [trl repository](https://github.com/huggingface/trl/tree/main/examples):\n\n| File | Task | Description | Colab link |\n|---|---| --- |\n| [`stack_llama/rl_training.py`](https://github.com/huggingface/trl/blob/main/examples/research_projects/stack_llama/scripts/rl_training.py) | RLHF | Distributed fine-tuning of the 7b parameter LLaMA models with a learned reward model and `peft`. |  |\n| [`stack_llama/reward_modeling.py`](https://github.com/huggingface/trl/blob/main/examples/research_projects/stack_llama/scripts/reward_modeling.py) | Reward Modeling | Distributed training of the 7b parameter LLaMA reward model with `peft`. |  |\n| [`stack_llama/supervised_finetuning.py`](https://github.com/huggingface/trl/blob/main/examples/research_projects/stack_llama/scripts/supervised_finetuning.py) | SFT | Distributed instruction/supervised fine-tuning of the 7b parameter LLaMA model with `peft`. |  |\n\n## Installation\nNote: peft is in active development, so we install directly from their Github page.\nPeft also relies on the latest version of transformers. \n\n```bash\npip install trl[peft]\npip install bitsandbytes loralib\npip install git+https://github.com/huggingface/transformers.git@main\n#optional: wandb\npip install wandb\n```\n\nNote: if you don't want to log with `wandb` remove `log_with=\"wandb\"` in the scripts/notebooks. You can also replace it with your favourite experiment tracker that's [supported by `accelerate`](https://huggingface.co/docs/accelerate/usage_guides/tracking).\n\n## How to use it?\n\nSimply declare a `PeftConfig` object in your script and pass it through `.from_pretrained` to load the TRL+PEFT model. \n\n```python\nfrom peft import LoraConfig\nfrom trl import AutoModelForCausalLMWithValueHead\n\nmodel_id = \"edbeeching/gpt-neo-125M-imdb\"\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(\n    model_id, \n    peft_config=lora_config,\n)\n```\nAnd if you want to load your model in 8bit precision:\n```python\npretrained_model = AutoModelForCausalLMWithValueHead.from_pretrained(\n    config.model_name, \n    load_in_8bit=True,\n    peft_config=lora_config,\n)\n```\n... or in 4bit precision:\n```python\npretrained_model = AutoModelForCausalLMWithValueHead.from_pretrained(\n    config.model_name, \n    peft_config=lora_config,\n    load_in_4bit=True,\n)\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 37eda51f-5e0b-4f80-b3e5-e41f31cbc028
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/lora_tuning_peft", "title": "Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)", "tokens": 1651, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"37eda51f-5e0b-4f80-b3e5-e41f31cbc028\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"591c3ecb-dff5-4b7a-a6bf-e34be2d4db07\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b85a18f377a4d66e4d735d15f1a65e6ee776244672acdc38b165a9ac70b9adfc\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3166baf8-6190-42e9-af07-d75387596112\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"6f1af7d9a1bd7c073ec263788119ea8b9cb7001bea6a4cf24f4a5066ed31afda\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"54f5209e-74ca-48a1-bf85-393d765f0be2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"99eb2aaeec1dd5c981bbbc92f450ded40a567b7497e2ffcb463257dc0b103296\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2913, \"end_char_idx\": 5857, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "ref_doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "chroma:document": "## Launch scripts\n\nThe `trl` library is powered by `accelerate`. As such it is best to configure and launch trainings with the following commands:\n\n```bash\naccelerate config # will prompt you to define the training configuration\naccelerate launch examples/scripts/ppo.py --use_peft # launch`es training\n```\n\n## Using `trl` + `peft` and Data Parallelism\n\nYou can scale up to as many GPUs as you want, as long as you are able to fit the training process in a single device. The only tweak you need to apply is to load the model as follows:\n```python\nfrom peft import LoraConfig\n...\n\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\npretrained_model = AutoModelForCausalLMWithValueHead.from_pretrained(\n    config.model_name, \n    peft_config=lora_config,\n)\n```\nAnd if you want to load your model in 8bit precision:\n```python\npretrained_model = AutoModelForCausalLMWithValueHead.from_pretrained(\n    config.model_name, \n    peft_config=lora_config,\n    load_in_8bit=True,\n)\n```\n... or in 4bit precision:\n```python\npretrained_model = AutoModelForCausalLMWithValueHead.from_pretrained(\n    config.model_name, \n    peft_config=lora_config,\n    load_in_4bit=True,\n)\n```\nFinally, make sure that the rewards are computed on correct device as well, for that you can use `ppo_trainer.model.current_device`.\n\n## Naive pipeline parallelism (NPP) for large models (>60B models)\n\nThe `trl` library also supports naive pipeline parallelism (NPP) for large models (>60B models). This is a simple way to parallelize the model across multiple GPUs. \nThis paradigm, termed as \"Naive Pipeline Parallelism\" (NPP) is a simple way to parallelize the model across multiple GPUs. We load the model and the adapters across multiple GPUs and the activations and gradients will be naively communicated across the GPUs. This supports `int8` models as well as other `dtype` models.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-npp.png\">\n</div>\n\n### How to use NPP?\n\nSimply load your model with a custom `device_map` argument on the `from_pretrained` to split your model across multiple devices. Check out this [nice tutorial](https://github.com/huggingface/blog/blob/main/accelerate-large-models.md) on how to properly create a `device_map` for your model. \n \nAlso make sure to have the `lm_head` module on the first GPU device as it may throw an error if it is not on the first device. As this time of writing, you need to install the `main` branch of `accelerate`: `pip install git+https://github.com/huggingface/accelerate.git@main` and `peft`: `pip install git+https://github.com/huggingface/peft.git@main`.\n\n### Launch scripts\n\nAlthough `trl` library is powered by `accelerate`, you should run your training script in a single process. Note that we do not support Data Parallelism together with NPP yet."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 54f5209e-74ca-48a1-bf85-393d765f0be2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/lora_tuning_peft", "title": "Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)", "tokens": 1651, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"54f5209e-74ca-48a1-bf85-393d765f0be2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"591c3ecb-dff5-4b7a-a6bf-e34be2d4db07\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b85a18f377a4d66e4d735d15f1a65e6ee776244672acdc38b165a9ac70b9adfc\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"37eda51f-5e0b-4f80-b3e5-e41f31cbc028\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/lora_tuning_peft\", \"title\": \"Examples of using peft with trl to finetune 8-bit models with Low Rank Adaption (LoRA)\", \"tokens\": 1651, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"224f7fd9820713a80a8a9bdefabf5cac5bc62423b2f9b3a772b823170cc1e6c1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4286, \"end_char_idx\": 6366, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "ref_doc_id": "591c3ecb-dff5-4b7a-a6bf-e34be2d4db07", "chroma:document": "## Naive pipeline parallelism (NPP) for large models (>60B models)\n\nThe `trl` library also supports naive pipeline parallelism (NPP) for large models (>60B models). This is a simple way to parallelize the model across multiple GPUs. \nThis paradigm, termed as \"Naive Pipeline Parallelism\" (NPP) is a simple way to parallelize the model across multiple GPUs. We load the model and the adapters across multiple GPUs and the activations and gradients will be naively communicated across the GPUs. This supports `int8` models as well as other `dtype` models.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl-npp.png\">\n</div>\n\n### How to use NPP?\n\nSimply load your model with a custom `device_map` argument on the `from_pretrained` to split your model across multiple devices. Check out this [nice tutorial](https://github.com/huggingface/blog/blob/main/accelerate-large-models.md) on how to properly create a `device_map` for your model. \n \nAlso make sure to have the `lm_head` module on the first GPU device as it may throw an error if it is not on the first device. As this time of writing, you need to install the `main` branch of `accelerate`: `pip install git+https://github.com/huggingface/accelerate.git@main` and `peft`: `pip install git+https://github.com/huggingface/peft.git@main`.\n\n### Launch scripts\n\nAlthough `trl` library is powered by `accelerate`, you should run your training script in a single process. Note that we do not support Data Parallelism together with NPP yet.\n\n```bash\npython PATH_TO_SCRIPT\n```\n\n## Fine-tuning Llama-2 model\n\nYou can easily fine-tune Llama2 model using `SFTTrainer` and the official script! For example to fine-tune llama2-7b on the Guanaco dataset, run (tested on a single NVIDIA T4-16GB):\n\n```bash\npython examples/scripts/sft.py --output_dir sft_openassistant-guanaco  --model_name meta-llama/Llama-2-7b-hf --dataset_name timdettmers/openassistant-guanaco --load_in_4bit --use_peft --per_device_train_batch_size 4 --gradient_accumulation_steps 2\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: ba7dce9a-64a6-4547-98ea-de157b2b0036
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/index", "title": "TRL - Transformer Reinforcement Learning", "tokens": 1175, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"ba7dce9a-64a6-4547-98ea-de157b2b0036\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/index\", \"title\": \"TRL - Transformer Reinforcement Learning\", \"tokens\": 1175, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"cce7cb1c-a5d7-488c-8b37-ac2278c3093b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/index\", \"title\": \"TRL - Transformer Reinforcement Learning\", \"tokens\": 1175, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2364d0692f57cac170f08daa81116f620f730f7bb7f5337e6e70550f7b0bc9a2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c170eebc-be36-4887-904f-f8981817e4de\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"95378f3bbc4c26f88ce00f96241fd5cd861ad507aea8a4e12b1a23e13a837ad0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2112, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "doc_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "ref_doc_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "chroma:document": "<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl_banner_dark.png\">\n</div>\n\n# TRL - Transformer Reinforcement Learning\n\nTRL is a full stack library where we provide a set of tools to train transformer language models with Reinforcement Learning, from the Supervised Fine-tuning step (SFT), Reward Modeling step (RM) to the Proximal Policy Optimization (PPO) step. \nThe library is integrated with \ud83e\udd17 [transformers](https://github.com/huggingface/transformers).\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/TRL-readme.png\">\n</div>\n\nCheck the appropriate sections of the documentation depending on your needs:\n\n## API documentation\n\n- [Model Classes](models): *A brief overview of what each public model class does.*\n- [`SFTTrainer`](sft_trainer): *Supervise Fine-tune your model easily with `SFTTrainer`*\n- [`RewardTrainer`](reward_trainer): *Train easily your reward model using `RewardTrainer`.*\n- [`PPOTrainer`](ppo_trainer): *Further fine-tune the supervised fine-tuned model using PPO algorithm*\n- [Best-of-N Sampling](best-of-n): *Use best of n sampling as an alternative way to sample predictions from your active model*\n- [`DPOTrainer`](dpo_trainer): *Direct Preference Optimization training using `DPOTrainer`.*\n- [`TextEnvironment`](text_environments): *Text environment to train your model using tools with RL.*\n\n## Examples\n\n- [Sentiment Tuning](sentiment_tuning): *Fine tune your model to generate positive movie contents*\n- [Training with PEFT](lora_tuning_peft): *Memory efficient RLHF training using adapters with PEFT*\n- [Detoxifying LLMs](detoxifying_a_lm): *Detoxify your language model through RLHF*\n- [StackLlama](using_llama_models): *End-to-end RLHF training of a Llama model on Stack exchange dataset*\n- [Learning with Tools](learning_tools): *Walkthrough of using `TextEnvironments`*\n- [Multi-Adapter Training](multi_adapter_rl): *Use a single base model and multiple adapters for memory efficient end-to-end training*"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: c170eebc-be36-4887-904f-f8981817e4de
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/index", "title": "TRL - Transformer Reinforcement Learning", "tokens": 1175, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"c170eebc-be36-4887-904f-f8981817e4de\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/index\", \"title\": \"TRL - Transformer Reinforcement Learning\", \"tokens\": 1175, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"cce7cb1c-a5d7-488c-8b37-ac2278c3093b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/index\", \"title\": \"TRL - Transformer Reinforcement Learning\", \"tokens\": 1175, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2364d0692f57cac170f08daa81116f620f730f7bb7f5337e6e70550f7b0bc9a2\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ba7dce9a-64a6-4547-98ea-de157b2b0036\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/index\", \"title\": \"TRL - Transformer Reinforcement Learning\", \"tokens\": 1175, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a541285210c1380a1728faa20fe326c542ffb37b5bc1135bfb26f5ce9d8b4f29\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2115, \"end_char_idx\": 4407, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "doc_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "ref_doc_id": "cce7cb1c-a5d7-488c-8b37-ac2278c3093b", "chroma:document": "## Blog posts\n\n<div class=\"mt-10\">\n  <div class=\"w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-2 md:gap-y-4 md:gap-x-5\">\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/dpo_vlm\">\n      <img src=\"https://raw.githubusercontent.com/huggingface/blog/main/assets/dpo_vlm/thumbnail.png\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">Preference Optimization for Vision Language Models with TRL</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/rlhf\">\n      <img src=\"https://raw.githubusercontent.com/huggingface/blog/main/assets/120_rlhf/thumbnail.png\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">Illustrating Reinforcement Learning from Human Feedback</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/trl-peft\">\n      <img src=\"https://github.com/huggingface/blog/blob/main/assets/133_trl_peft/thumbnail.png?raw=true\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">Fine-tuning 20B LLMs with RLHF on a 24GB consumer GPU</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/stackllama\">\n      <img src=\"https://github.com/huggingface/blog/blob/main/assets/138_stackllama/thumbnail.png?raw=true\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">StackLLaMA: A hands-on guide to train LLaMA with RLHF</p>\n   </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/dpo-trl\">\n      <img src=\"https://github.com/huggingface/blog/blob/main/assets/157_dpo_trl/dpo_thumbnail.png?raw=true\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">Fine-tune Llama 2 with DPO</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"https://huggingface.co/blog/trl-ddpo\">\n      <img src=\"https://github.com/huggingface/blog/blob/main/assets/166_trl_ddpo/thumbnail.png?raw=true\" alt=\"thumbnail\">\n      <p class=\"text-gray-700\">Finetune Stable Diffusion Models with DDPO via TRL</p>\n    </a>\n  </div>\n</div>"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3c83377e-213c-4333-ac50-be169c4cd020
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/bco_trainer", "title": "BCO Trainer", "tokens": 1123, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3c83377e-213c-4333-ac50-be169c4cd020\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/bco_trainer\", \"title\": \"BCO Trainer\", \"tokens\": 1123, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ec276d4-0b0d-4707-bd4f-3e4707fd73e7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/bco_trainer\", \"title\": \"BCO Trainer\", \"tokens\": 1123, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"e7eab5ea553608682e479c1adcb2564188810fadf8a29b77acb18f877a62fc83\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b62359f6-1c12-4db7-8c42-0c8e0ba64c61\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3fb9a961376776e789897b24c9dcfb27586a66157739a4a39347c63ed2c39193\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2565, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "doc_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "ref_doc_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "chroma:document": "# BCO Trainer\n\nTRL supports the Binary Classifier Optimization (BCO).\nThe [BCO](https://huggingface.co/papers/2404.04656) authors train a binary classifier whose logit serves as a reward so that the classifier maps {prompt, chosen completion} pairs to 1 and {prompt, rejected completion} pairs to 0.\nFor a full example have a look at  [`examples/scripts/bco.py`].\n\n## Expected dataset format\n\nThe BCO trainer expects a very specific format for the dataset as it does not require pairwise preferences. Since the model will be trained to directly optimize examples that consist of a prompt, model completion, and a label to indicate whether the completion is \"good\" or \"bad\", we expect a dataset with the following columns:\n\n- `prompt`\n- `completion`\n- `label`\n\nfor example:\n\n```\nbco_dataset_dict = {\n    \"prompt\": [\n        \"Hey, hello\",\n        \"How are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"completion\": [\n        \"hi nice to meet you\",\n        \"leave me alone\",\n        \"I don't have a name\",\n        \"My name is Mary\",\n        \"Python\",\n        \"C++\",\n        \"Java\",\n    ],\n    \"label\": [\n        True,\n        False,\n        False,\n        True,\n        True,\n        False,\n        False,\n    ],\n}\n```\n\nwhere the `prompt` contains the context inputs, `completion` contains the corresponding responses and `label` contains the corresponding flag that indicates if the generated completion is desired (`True`) or undesired (`False`).\nA prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays. It is required that the dataset contains at least one desirable and one undesirable completion.\n\n\n## Expected model format\nThe BCO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `BCOTrainer`\n\nFor a detailed example have a look at the `examples/scripts/bco.py` script. At a high level we need to initialize the `BCOTrainer` with a `model` we wish to train and a reference `ref_model` which we will use to calculate the implicit rewards of the preferred and rejected response. \n\nThe `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above. Note that the `model` and `ref_model` need to have the same architecture (ie decoder only or encoder-decoder)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b62359f6-1c12-4db7-8c42-0c8e0ba64c61
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/bco_trainer", "title": "BCO Trainer", "tokens": 1123, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b62359f6-1c12-4db7-8c42-0c8e0ba64c61\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/bco_trainer\", \"title\": \"BCO Trainer\", \"tokens\": 1123, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ec276d4-0b0d-4707-bd4f-3e4707fd73e7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/bco_trainer\", \"title\": \"BCO Trainer\", \"tokens\": 1123, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"e7eab5ea553608682e479c1adcb2564188810fadf8a29b77acb18f877a62fc83\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c83377e-213c-4333-ac50-be169c4cd020\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/bco_trainer\", \"title\": \"BCO Trainer\", \"tokens\": 1123, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"e68719b718cce2b0bc378bb8600165964756f3c0c180ce9a248665fc0d0ca41e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1852, \"end_char_idx\": 4993, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "doc_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "ref_doc_id": "9ec276d4-0b0d-4707-bd4f-3e4707fd73e7", "chroma:document": "## Expected model format\nThe BCO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `BCOTrainer`\n\nFor a detailed example have a look at the `examples/scripts/bco.py` script. At a high level we need to initialize the `BCOTrainer` with a `model` we wish to train and a reference `ref_model` which we will use to calculate the implicit rewards of the preferred and rejected response. \n\nThe `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above. Note that the `model` and `ref_model` need to have the same architecture (ie decoder only or encoder-decoder).\n\n\n\n```py\ntraining_args = BCOConfig(\n    beta=0.1,\n)\n\nbco_trainer = BCOTrainer(\n    model,\n    model_ref,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\nbco_trainer.train()\n```\n\n## Underlying Distribution matching (UDM)\n\nIn practical scenarios, the thumbs-up and thumbs-down datasets are likely to have divergent underlying distributions of prompts.\nConsider an LLM deployed for user feedback: if the model excels in writing tasks but underperforms in coding, the thumbs-up dataset will be dominated by writing-related prompts, while the thumbs-down dataset will contain mostly coding-related prompts.  \nIf the prompts in your desired and undesired datasets differ a lot, it is useful to enable UDM.  \n\nChoose an embedding model and tokenizer:\n\n```py\nembedding_model = AutoModel.from_pretrained(your_model_id)\nembedding_tokenizer = AutoTokenizer.from_pretrained(your_model_id)\n\n# customize this function depending on your embedding model\ndef embed_prompt(input_ids, attention_mask, model):\n    outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n    return outputs.last_hidden_state.mean(dim=1)\n\nembedding_model = Accelerator().prepare_model(self.embedding_model)\nembedding_func = partial(embed_prompt, model=embedding_model)\n```\n\nSet `prompt_sample_size` to defined how many prompts are selected to train the UDM classifier and start the training with the provided embedding function:\n\n```py\ntraining_args = BCOConfig(\n    beta=0.1,\n    prompt_sample_size=512,\n)\n\nbco_trainer = BCOTrainer(\n    model,\n    model_ref,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n    embedding_func=embedding_func,\n    embedding_tokenizer=self.embedding_tokenizer,\n)\n\nbco_trainer.train()\n```\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.  \n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001).\n\n## BCOTrainer\n\n[[autodoc]] BCOTrainer\n\n## BCOConfig\n\n[[autodoc]] BCOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a7aeda38-2d51-44cf-83f2-cecd015d2fa7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a7aeda38-2d51-44cf-83f2-cecd015d2fa7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d0f98b80-0069-4476-93e2-fedffbe17b48\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3e00ce8c0cf1ca016317a646cdb6950d97bebb3e6700d87fff44435b1421bfa1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1753, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "# RLOO Trainer\n\nTRL supports training LLMs with REINFORCE Leave-One-Out (RLOO). The idea is that instead of using a value function, RLOO generates K completions for each prompt. For each completion, RLOO uses the mean scores from the other K-1 completions as a baseline to calculate the advantage. RLOO also models the entire completion as a single action, where as PPO models each token as an action. Note that REINFORCE / A2C is a special case of PPO, when the number of PPO epochs is 1 and the number of mini-batches is 1, which is how we implement RLOO in TRL.\n\nReferences:\n- [Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs](https://huggingface.co/papers/2402.14740)\n- [A2C is a special case of PPO](https://huggingface.co/papers/2205.09123)\n- [Fine-Tuning Language Models from Human Preferences](https://github.com/openai/lm-human-preferences)\n- [Learning to Summarize from Human Feedback](https://github.com/openai/summarize-from-feedback)\n- [The N Implementation Details of RLHF with PPO](https://huggingface.co/blog/the_n_implementation_details_of_rlhf_with_ppo)\n- [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031)\n\n## Get started\n\nTo just run a RLOO script to make sure the trainer can run, you can run the following command to train a RLOO model with a dummy reward model.\n\n```bash\npython examples/scripts/rloo/rloo.py \\\n    --learning_rate 3e-6 \\\n    --output_dir models/minimal/rloo \\\n    --per_device_train_batch_size 64 \\\n    --gradient_accumulation_steps 1 \\\n    --total_episodes 10000 \\\n    --model_name_or_path EleutherAI/pythia-14m \\\n    --reward_model_path EleutherAI/pythia-14m \\\n    --non_eos_penalty\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d0f98b80-0069-4476-93e2-fedffbe17b48
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d0f98b80-0069-4476-93e2-fedffbe17b48\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a7aeda38-2d51-44cf-83f2-cecd015d2fa7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"5c5fbb0a55ceb9e126632d97e9d07279251773c46243a22d5693f3e44356c371\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b8387649-6f01-403d-8e9e-1eb50b1f3ec9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d4cd8eb6822f89a0377662f3ad5d11559b1b4c2dfd32b2973da41674011b4cee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1756, \"end_char_idx\": 4005, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "## Explanation of the logged metrics\n\nThe logged metrics are as follows. Here is an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/u2sqci34)\n\n<!-- * `rlhf_reward_var_per_prompt`: calculated by `rlhf_reward.var(0).mean()`. This is the variance of the rewards estimated across the `args.rloo_k` samples. Usually we expect it to go down (cause policy entropy goes down). -->\n\n* `eps`: Tracks the number of episodes per second.\n* `objective/kl`: The mean Kullback-Leibler (KL) divergence between the current policy and reference policy.\n* `objective/entropy`: The mean entropy of the policy, indicating the randomness of the actions chosen by the policy.\n* `objective/non_score_reward`: The mean reward from non-score-related sources, basically `beta * kl.sum(1)`, where `beta` is the KL penalty coefficient and `kl` is the per-token KL divergence.\n* `objective/rlhf_reward`: The mean RLHF reward, which is `score - non_score_reward`.\n* `objective/scores`: The mean scores returned by the reward model / environment.\n* `policy/approxkl_avg`: The average approximate KL divergence between consecutive PPO policies. Note that this is not the same as `objective/kl`.\n* `policy/clipfrac_avg`: The average fraction of policy updates that are clipped, indicating how often the policy updates are constrained to prevent large changes.\n* `loss/policy_avg`: The average policy loss, indicating how well the policy is performing.\n* `val/clipfrac_avg`: The average fraction of value function updates that are clipped, similar to policy/clipfrac_avg but for the value function.\n* `policy/entropy_avg`: The average entropy of the policy during training, indicating how diverse the policy's actions are.\n* `val/ratio`: The mean ratio of the current policy probability to the old policy probability, providing a measure of how much the policy has changed.\n* `val/ratio_var`: The variance of the `val/ratio`, indicating the variability in policy changes.\n* `val/num_eos_tokens`: The number of end-of-sequence (EOS) tokens generated, which can indicate the number of complete responses.\n* `lr`: lr: The current learning rate used by the optimizer.\n* `episode`: episode: The current global step or episode count in the training process."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b8387649-6f01-403d-8e9e-1eb50b1f3ec9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b8387649-6f01-403d-8e9e-1eb50b1f3ec9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d0f98b80-0069-4476-93e2-fedffbe17b48\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"335e96bad1f832e53115551effe749319bcc1223902f129689caec2d361f2972\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a59174f1-0b0f-4fb4-9e5b-a94cbf96338d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a15e25646b7c15b6d7e0e0205e85014d4c5074e67b488c95f4d0faa98345ae0e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4008, \"end_char_idx\": 7273, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "## Cookbook\n\n* Debugging TIP: `objective/rlhf_reward`: this is the ultimate objective of the RLHF training. If training works as intended, this metric should keep going up.\n* Debugging TIP: `val/ratio`: this number should float around 1.0, and it gets clipped by `--cliprange 0.2` with PPO's surrogate loss. So if this `ratio` is too high like 2.0 or 1000.0 or too small like 0.1, it means the updates between consecutive policies are too drastic. You should try undertand why this is happening and try to fix it.\n* Memory TIP: If you are running out of memory, you can try to reduce the `--per_device_train_batch_size` or increase the `--gradient_accumulation_steps` to reduce the memory footprint.\n* Memory TIP: If you have multiple GPUs, you can also run training with DeepSpeed stage 3 to reduce the memory footprint `accelerate launch --config_file examples/accelerate_configs/deepspeed_zero3.yaml`.\n* Usage TIP: We recommend to use the \"EOS trick\" via `--non_eos_penalty --stop_token eos`, which replaces the score of completions that do not end with an EOS token with a static scalar penalty `--penalty_reward_value`. This can help the model learn to generate more coherent completions.\n\n\n## What is my model doing exactly?\n\nTo help you understand what your model is doing, we periodically log some sample completions from the model. Here is an example of a completion. In an example [tracked run at Weights and Biases](https://wandb.ai/huggingface/trl/runs/u2sqci34), it looks like the following, allowing you to see the model's response at different stages of training. By default we generate `--num_sample_generations 10` during training, but you can customize the number of generations.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/ppov2_completions.gif)\n\n\nIn the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?               \u2502 desperate.<|endoftext|>[PAD][P\u2026 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 POST: Hi,                       \u2502                                 \u2502          \u2502\n\u2502 I'm 22, and I have been with my \u2502                                 \u2502          \u2502\n\u2502 girlfriend for 5 years now. We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Problem, I recently started to  \u2502                                 \u2502          \u2502\n\u2502 have feelings for an other      \u2502                                 \u2502          \u2502\n\u2502 person (a friend)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a59174f1-0b0f-4fb4-9e5b-a94cbf96338d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a59174f1-0b0f-4fb4-9e5b-a94cbf96338d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b8387649-6f01-403d-8e9e-1eb50b1f3ec9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"04e5bcdf45b423fb542404d7a7faae88060e22b43c8323043a5c3bc20c52e865\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f1f189b2-9e92-450d-9c0b-af11c5384f83\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"30a60f1fce9c5beba5db830dfeae9ab17ec87e98143f953a48aeb86e5ec03365\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5824, \"end_char_idx\": 9560, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "In the logs the sampled generations look like \n\n```\n\u250f\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2533\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2513\n\u2503 query                           \u2503 model response                  \u2503 score    \u2503\n\u2521\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2547\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2529\n\u2502  SUBREDDIT: r/AskReddit         \u2502  I'm in love with a friend, and \u2502 3.921875 \u2502\n\u2502                                 \u2502 I don't know how to get rid of  \u2502          \u2502\n\u2502 TITLE: How do you get someone   \u2502 those feelings. I'm             \u2502          \u2502\n\u2502 out of your head?               \u2502 desperate.<|endoftext|>[PAD][P\u2026 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 POST: Hi,                       \u2502                                 \u2502          \u2502\n\u2502 I'm 22, and I have been with my \u2502                                 \u2502          \u2502\n\u2502 girlfriend for 5 years now. We  \u2502                                 \u2502          \u2502\n\u2502 recently moved together. We've  \u2502                                 \u2502          \u2502\n\u2502 always loved each other         \u2502                                 \u2502          \u2502\n\u2502 intensely.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Problem, I recently started to  \u2502                                 \u2502          \u2502\n\u2502 have feelings for an other      \u2502                                 \u2502          \u2502\n\u2502 person (a friend). This person  \u2502                                 \u2502          \u2502\n\u2502 has had a boyfriend for now 3   \u2502                                 \u2502          \u2502\n\u2502 years, and has absolutely no    \u2502                                 \u2502          \u2502\n\u2502 ideas. Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV."}


