# embeddings_queue

created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: f1f189b2-9e92-450d-9c0b-af11c5384f83
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"f1f189b2-9e92-450d-9c0b-af11c5384f83\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a59174f1-0b0f-4fb4-9e5b-a94cbf96338d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9a3ca3d3fe11e82b59ef7dc3c9f273a0bba38d42245587df6db2081bffbf8395\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a6633b45-cd84-4807-86ef-369834879768\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"09a574467369a662392b762f3593804cdf0a5b272606286a892569cb5ecefa89\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7505, \"end_char_idx\": 11326, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "Those feelings were so   \u2502                                 \u2502          \u2502\n\u2502 strong, it was hard to hide     \u2502                                 \u2502          \u2502\n\u2502 them. After 2 months of me      \u2502                                 \u2502          \u2502\n\u2502 being distant and really sad,   \u2502                                 \u2502          \u2502\n\u2502 my girlfriend forced me to say  \u2502                                 \u2502          \u2502\n\u2502 what was bothering me. I'm not  \u2502                                 \u2502          \u2502\n\u2502 a good liar, and now she knows. \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 We decided to give us a week    \u2502                                 \u2502          \u2502\n\u2502 alone, I went to my parents.    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 Now, I'm completely lost. I     \u2502                                 \u2502          \u2502\n\u2502 keep on thinking about this     \u2502                                 \u2502          \u2502\n\u2502 person, and I hate that. I      \u2502                                 \u2502          \u2502\n\u2502 would like for those feelings   \u2502                                 \u2502          \u2502\n\u2502 to go away, to leave me alone.  \u2502                                 \u2502          \u2502\n\u2502 But I can't.                    \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 What do I do? It's been 3       \u2502                                 \u2502          \u2502\n\u2502 months now, and I'm just        \u2502                                 \u2502          \u2502\n\u2502 desperate.                      \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n\u2502  SUBREDDIT: r/pettyrevenge      \u2502  My mom woke me up with a loud  \u2502 6.84375  \u2502\n\u2502                                 \u2502 TV. I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down. She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a6633b45-cd84-4807-86ef-369834879768
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a6633b45-cd84-4807-86ef-369834879768\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f1f189b2-9e92-450d-9c0b-af11c5384f83\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"5095636f8e0c4ba66f3197d5629a12633d4a0780f2987215632a0391c13b3524\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9e24a4da-0697-4049-8ba0-be10cbe72ff5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9b3bac5744478e9d6d2b3ca7b528f4694443d21bf0c56248d5a9eb9bea6aa46c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9561, \"end_char_idx\": 13651, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "I blasted Gangnam Style on  \u2502          \u2502\n\u2502 TITLE: So, my mom woke me up    \u2502 repeat, with the bass cranked   \u2502          \u2502\n\u2502 with a loud TV.                 \u2502 up as high as it could          \u2502          \u2502\n\u2502                                 \u2502 go.<|endoftext|>[PAD][PAD][PAD\u2026 \u2502          \u2502\n\u2502 POST: She was in her living     \u2502                                 \u2502          \u2502\n\u2502 room, watching TV. This was at  \u2502                                 \u2502          \u2502\n\u2502 about 8:30 in the morning, and  \u2502                                 \u2502          \u2502\n\u2502 she was exercising. She turned  \u2502                                 \u2502          \u2502\n\u2502 the TV up extra loud to hear it \u2502                                 \u2502          \u2502\n\u2502 over her excercycle, and woke   \u2502                                 \u2502          \u2502\n\u2502 me up. I went in there asking   \u2502                                 \u2502          \u2502\n\u2502 for her to turn it down. She    \u2502                                 \u2502          \u2502\n\u2502 said she didn't have to; I      \u2502                                 \u2502          \u2502\n\u2502 explained that I always used    \u2502                                 \u2502          \u2502\n\u2502 headphones so she didn't have   \u2502                                 \u2502          \u2502\n\u2502 to deal with my noise and that  \u2502                                 \u2502          \u2502\n\u2502 she should give me a little     \u2502                                 \u2502          \u2502\n\u2502 more respect, given that I paid \u2502                                 \u2502          \u2502\n\u2502 rent at the time.               \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 She disagreed. I went back to   \u2502                                 \u2502          \u2502\n\u2502 my room, rather pissed off at   \u2502                                 \u2502          \u2502\n\u2502 the lack of equality. I had no  \u2502                                 \u2502          \u2502\n\u2502 lock on my door; but I had a    \u2502                                 \u2502          \u2502\n\u2502 dresser right next to it, so I  \u2502                                 \u2502          \u2502\n\u2502 pulled one of the drawers out   \u2502                                 \u2502          \u2502\n\u2502 enough so that it caused the    \u2502                                 \u2502          \u2502\n\u2502 door to not be openable. Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it. But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nThe bulk of RLOOTrainer is based on the PPO implementation, which is based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 9e24a4da-0697-4049-8ba0-be10cbe72ff5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"9e24a4da-0697-4049-8ba0-be10cbe72ff5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a6633b45-cd84-4807-86ef-369834879768\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a1fb55c4397402ae6dac3dfb5b49eee9e71a5bbc1a538a8d99a4ebfb19b7644c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"43035130-54d7-42e8-ac2c-f23f56d25b27\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dfca07fb8a0cbac8cbb4420be363503b69f0ecd39ade2c359eb870be1dce18a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11735, \"end_char_idx\": 15231, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "Then,  \u2502                                 \u2502          \u2502\n\u2502 I turned my speakers up really  \u2502                                 \u2502          \u2502\n\u2502 loud and blasted Gangnam Style  \u2502                                 \u2502          \u2502\n\u2502 on repeat, with the bass        \u2502                                 \u2502          \u2502\n\u2502 cranked up as high as it could  \u2502                                 \u2502          \u2502\n\u2502 go.                             \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 If you hate Gangnam Style for   \u2502                                 \u2502          \u2502\n\u2502 being overplayed, you will see  \u2502                                 \u2502          \u2502\n\u2502 why I chose that particular     \u2502                                 \u2502          \u2502\n\u2502 song. I personally don't mind   \u2502                                 \u2502          \u2502\n\u2502 it. But here's the thing about  \u2502                                 \u2502          \u2502\n\u2502 my bass; it vibrates the walls, \u2502                                 \u2502          \u2502\n\u2502 making one hell of a lot of     \u2502                                 \u2502          \u2502\n\u2502 noise. Needless to say, my mom  \u2502                                 \u2502          \u2502\n\u2502 was not pleased and shut off    \u2502                                 \u2502          \u2502\n\u2502 the internet. But it was oh so  \u2502                                 \u2502          \u2502\n\u2502 worth it.                       \u2502                                 \u2502          \u2502\n\u2502                                 \u2502                                 \u2502          \u2502\n\u2502 TL;DR:                          \u2502                                 \u2502          \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Implementation details\n\nThe bulk of RLOOTrainer is based on the PPO implementation, which is based on the [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n\nBelow is a vectorized advantage calculation for RLOO:\n\n```python\ndef test_rloo_reward():\n    local_batch_size = 3\n    rloo_k = 4\n    rlhf_reward = torch.tensor([\n        1, 2, 3, # first rlhf reward for three prompts\n        2, 3, 4, # second rlhf reward for three prompts\n        5, 6, 7, # third rlhf reward for three prompts\n        8, 9, 10, # fourth rlhf reward for three prompts\n    ]).float() # here we have 3 prompts which have 4 completions each\n\n    baseline = (rlhf_reward.sum(0) - rlhf_reward) / (rloo_k - 1)\n    advantages = torch.zeros_like(rlhf_reward)\n    for i in range(0, len(advantages), local_batch_size):\n        other_response_rlhf_rewards = []\n        for j in range(0, len(advantages), local_batch_size):\n            if i != j:\n                other_response_rlhf_rewards.append(rlhf_reward[j : j + local_batch_size])\n        advantages[i : i + local_batch_size] = rlhf_reward[i : i + local_batch_size] - torch.stack(other_response_rlhf_rewards).mean(0)\n    \n    assert (1 - (2 + 5 + 8) / 3 - advantages[0].item()) < 1e-6  # First rlhf reward for the first prompt\n    assert (6 - (3 + 2 + 9) / 3 - advantages[7].item()) < 1e-6  # Third rlhf reward for the second prompt\n\n    # Vectorized implementation\n    rlhf_reward = rlhf_reward.reshape(rloo_k, local_batch_size)\n    baseline = (rlhf_reward.sum(0) - rlhf_reward) / (rloo_k - 1)\n    vec_advantages = rlhf_reward - baseline\n    torch.testing.assert_close(vec_advantages.flatten(), advantages)\n```\n\n## Benchmark experiments\n\nTo validate the RLOO implementation works, we ran experiment on the 1B model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 43035130-54d7-42e8-ac2c-f23f56d25b27
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"43035130-54d7-42e8-ac2c-f23f56d25b27\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9e24a4da-0697-4049-8ba0-be10cbe72ff5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"519f1936d2da001e54e6b9755cf3a4fa4f3364a2efdba06027b8f773f36e3df7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"20ef778f-8922-4f56-a905-28d9ac54ace2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c58aab0c83690639c86b81fe2951de02078877f6f9f6138b540ebce6b86345bc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15232, \"end_char_idx\": 17226, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "Here are the command we used to run the experiment. We take the SFT / RM models directly from [The N+ Implementation Details of RLHF with PPO: A Case Study on TL;DR Summarization](https://huggingface.co/papers/2403.17031).\n\n```\naccelerate launch --config_file examples/accelerate_configs/deepspeed_zero2.yaml \\\n    examples/scripts/rloo/rloo_tldr.py \\\n    --output_dir models/minimal/rloo_tldr \\\n    --num_ppo_epochs 2 \\\n    --num_mini_batches 2 \\\n    --learning_rate 3e-6 \\\n    --per_device_train_batch_size 8 \\\n    --gradient_accumulation_steps 8 \\\n    --total_episodes 1000000 \\\n    --model_name_or_path EleutherAI/pythia-1b-deduped \\\n    --sft_model_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr \\\n    --reward_model_path cleanrl/EleutherAI_pythia-1b-deduped__reward__tldr \\\n    --local_rollout_forward_batch_size 16 \\\n    --non_eos_penalty \\\n    --stop_token eos \\\n    --kl_coef 0.03\n```\n\nCheckpoints and experiment tracking are available at:\n\n- [\ud83e\udd17 Model checkpoint](https://huggingface.co/vwxyzjn/rloo_tldr)\n- [\ud83d\udc1d Tracked experiment](https://wandb.ai/huggingface/trl/runs/u2sqci34)\n\n\nTo evaluate, we use [vLLM](https://github.com/vllm-project/vllm) to load the checkpoints and GPT-4o mini as a judge model to evaluate the generated TL;DR against the reference TL;DR.\nFor more information on how to use judges, see [Judges](judges).\n\n```bash\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 33.00%\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/rloo_tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 51.20%\n```\n\nThe RLOO checkpoint gets a 51.2% preferred rate vs the 33.0% preference rate of the SFT checkpoint. This is a good sign that the RLOO training is working as intended.\n\n\nMetrics:\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/pr-1540/rloo.png)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 20ef778f-8922-4f56-a905-28d9ac54ace2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/rloo_trainer", "title": "RLOO Trainer", "tokens": 4313, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"20ef778f-8922-4f56-a905-28d9ac54ace2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f8b203be-3a43-463d-885c-7602ab0a4f39\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5cf80194ef172d4267b997a4d0e6e7d7cf784b689b5a8ec97cd465153319e31\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"43035130-54d7-42e8-ac2c-f23f56d25b27\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/rloo_trainer\", \"title\": \"RLOO Trainer\", \"tokens\": 4313, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"6d4480857deb71ca90db9bf909cbad75037aca04d26a7117155a447cd204419a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16328, \"end_char_idx\": 18286, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "ref_doc_id": "f8b203be-3a43-463d-885c-7602ab0a4f39", "chroma:document": "To evaluate, we use [vLLM](https://github.com/vllm-project/vllm) to load the checkpoints and GPT-4o mini as a judge model to evaluate the generated TL;DR against the reference TL;DR.\nFor more information on how to use judges, see [Judges](judges).\n\n```bash\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 33.00%\n$ python examples/scripts/evals/judge_tldr.py --model_name_or_path vwxyzjn/rloo_tldr --judge_model gpt-4o-mini --num_examples 1000\nModel win rate: 51.20%\n```\n\nThe RLOO checkpoint gets a 51.2% preferred rate vs the 33.0% preference rate of the SFT checkpoint. This is a good sign that the RLOO training is working as intended.\n\n\nMetrics:\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/pr-1540/rloo.png)\n\n\n```bash\n# pip install openrlbenchmark==0.2.1a5\n# see https://github.com/openrlbenchmark/openrlbenchmark#get-started for documentation\n# to use it, change `?we=huggingface&wpn=trl` to your own project and `?tag=pr-1540` to your own tag\npython -m openrlbenchmark.rlops_multi_metrics \\\n    --filters '?we=huggingface&wpn=trl&xaxis=train/episode&ceik=output_dir&cen=sft_model_path&metrics=train/objective/rlhf_reward&metrics=train/objective/scores&metrics=train/objective/kl&metrics=train/objective/non_score_reward&metrics=train/objective/entropy&metrics=train/policy/approxkl_avg&metrics=train/policy/clipfrac_avg&metrics=train/loss/policy_avg&metrics=train/policy/entropy_avg&metrics=train/val/ratio&metrics=train/val/ratio_var&metrics=train/val/num_eos_tokens&metrics=train/lr&metrics=train/eps' \\\n        \"cleanrl/EleutherAI_pythia-1b-deduped__sft__tldr?tag=pr-1540\" \\\n    --env-ids models/minimal/rloo_tldr \\\n    --pc.ncols 4 \\\n    --pc.ncols-legend 1 \\\n    --pc.xlabel \"Episode\" \\\n    --output-filename benchmark/trl/pr-1540/rloo \\\n    --scan-history\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 68fd38a7-3cf6-4a96-81ce-20fff0f5cdbc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/models", "title": "Models", "tokens": 228, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"68fd38a7-3cf6-4a96-81ce-20fff0f5cdbc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/models\", \"title\": \"Models\", \"tokens\": 228, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b92412e8-9dd2-420c-a369-24475992b889\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/models\", \"title\": \"Models\", \"tokens\": 228, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"522d880cd267eff65a257f850cd975795581fefc5e616e1bf662e468d6c7cf47\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 906, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b92412e8-9dd2-420c-a369-24475992b889", "doc_id": "b92412e8-9dd2-420c-a369-24475992b889", "ref_doc_id": "b92412e8-9dd2-420c-a369-24475992b889", "chroma:document": "# Models\n\nWith the `AutoModelForCausalLMWithValueHead` class TRL supports all decoder model architectures in transformers such as GPT-2, OPT, and GPT-Neo. In addition, with `AutoModelForSeq2SeqLMWithValueHead` you can use encoder-decoder architectures such as T5. TRL also requires reference models which are frozen copies of the model that is trained. With `create_reference_model` you can easily create a frozen copy and also share layers between the two models to save memory.\n\n## PreTrainedModelWrapper\n\n[[autodoc]] PreTrainedModelWrapper\n\n## AutoModelForCausalLMWithValueHead\n\n\n[[autodoc]] AutoModelForCausalLMWithValueHead\n    - __init__\n    - forward\n    - generate\n    - _init_weights\n\n## AutoModelForSeq2SeqLMWithValueHead\n\n[[autodoc]] AutoModelForSeq2SeqLMWithValueHead\n    - __init__\n    - forward\n    - generate\n    - _init_weights\n\n## create_reference_model\n\n[[autodoc]] create_reference_model"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 4cca7bc5-8c18-4d1f-a9b6-87d29c1a6756
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/use_model", "title": "Use model after training", "tokens": 599, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"4cca7bc5-8c18-4d1f-a9b6-87d29c1a6756\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/use_model\", \"title\": \"Use model after training\", \"tokens\": 599, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b4019dfd-8013-4d4e-a9e1-3e47967679c5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/use_model\", \"title\": \"Use model after training\", \"tokens\": 599, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"89df1a661d4ff88fd4778139116cadcf281438f176dbce081740b68670b2445c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2559, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b4019dfd-8013-4d4e-a9e1-3e47967679c5", "doc_id": "b4019dfd-8013-4d4e-a9e1-3e47967679c5", "ref_doc_id": "b4019dfd-8013-4d4e-a9e1-3e47967679c5", "chroma:document": "# Use model after training\n\nOnce you have trained a model using either the SFTTrainer, PPOTrainer, or DPOTrainer, you will have a fine-tuned model that can be used for text generation. In this section, we'll walk through the process of loading the fine-tuned model and generating text. If you need to run an inference server with the trained model, you can explore libraries such as [`text-generation-inference`](https://github.com/huggingface/text-generation-inference).\n\n## Load and Generate\n\nIf you have fine-tuned a model fully, meaning without the use of PEFT you can simply load it like any other language model in transformers. E.g. the value head that was trained during the PPO training is no longer needed and if you load the model with the original transformer class it will be ignored:\n\n```python\nfrom transformers import AutoTokenizer, AutoModelForCausalLM\n\nmodel_name_or_path = \"kashif/stack-llama-2\" #path/to/your/model/or/name/on/hub\ndevice = \"cpu\" # or \"cuda\" if you have a GPU\n\nmodel = AutoModelForCausalLM.from_pretrained(model_name_or_path).to(device)\ntokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n\ninputs = tokenizer.encode(\"This movie was really\", return_tensors=\"pt\").to(device)\noutputs = model.generate(inputs)\nprint(tokenizer.decode(outputs[0]))\n```\n\nAlternatively you can also use the pipeline:\n\n```python\nfrom transformers import pipeline\n\nmodel_name_or_path = \"kashif/stack-llama-2\" #path/to/your/model/or/name/on/hub\npipe = pipeline(\"text-generation\", model=model_name_or_path)\nprint(pipe(\"This movie was really\")[0][\"generated_text\"])\n```\n\n## Use Adapters PEFT\n\n```python\nfrom peft import PeftConfig, PeftModel\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\nbase_model_name = \"kashif/stack-llama-2\" #path/to/your/model/or/name/on/hub\"\nadapter_model_name = \"path/to/my/adapter\"\n\nmodel = AutoModelForCausalLM.from_pretrained(base_model_name)\nmodel = PeftModel.from_pretrained(model, adapter_model_name)\n\ntokenizer = AutoTokenizer.from_pretrained(base_model_name)\n```\n\nYou can also merge the adapters into the base model so you can use the model like a normal transformers model, however the checkpoint will be significantly bigger:\n\n```python\nmodel = AutoModelForCausalLM.from_pretrained(base_model_name)\nmodel = PeftModel.from_pretrained(model, adapter_model_name)\n\nmodel = model.merge_and_unload()\nmodel.save_pretrained(\"merged_adapters\")\n```\n\nOnce you have the model loaded and either merged the adapters or keep them separately on top you can run generation as with a normal model outlined above."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 457acb02-0b6c-4be0-9a39-0e624c8439e7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/trainer", "title": "Trainer", "tokens": 371, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"457acb02-0b6c-4be0-9a39-0e624c8439e7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/trainer\", \"title\": \"Trainer\", \"tokens\": 371, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"32c1ee8a-ed50-419b-94a3-b03846b3a9bb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/trainer\", \"title\": \"Trainer\", \"tokens\": 371, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c7cece391a0ec34263b40a2f3a8f8a98f67f4e62d953f45f1caba607aeb1e650\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1188, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "32c1ee8a-ed50-419b-94a3-b03846b3a9bb", "doc_id": "32c1ee8a-ed50-419b-94a3-b03846b3a9bb", "ref_doc_id": "32c1ee8a-ed50-419b-94a3-b03846b3a9bb", "chroma:document": "# Trainer\n\nAt TRL we support PPO (Proximal Policy Optimisation) with an implementation that largely follows  the structure introduced in the paper \"Fine-Tuning Language Models from Human Preferences\" by D. Ziegler et al. [[paper](https://huggingface.co/papers/1909.08593), [code](https://github.com/openai/lm-human-preferences)].\nThe Trainer and model classes are largely inspired from `transformers.Trainer` and `transformers.AutoModel` classes and adapted for RL.\nWe also support a `RewardTrainer` that can be used to train a reward model.\n\n\n## CPOConfig\n\n[[autodoc]] CPOConfig\n\n## CPOTrainer\n\n[[autodoc]] CPOTrainer\n\n## DDPOConfig\n\n[[autodoc]] DDPOConfig\n\n## DDPOTrainer\n\n[[autodoc]] DDPOTrainer\n\n## DPOTrainer\n\n[[autodoc]] DPOTrainer\n\n## IterativeSFTTrainer\n\n[[autodoc]] IterativeSFTTrainer\n\n## KTOConfig\n\n[[autodoc]] KTOConfig\n\n## KTOTrainer\n\n[[autodoc]] KTOTrainer\n\n## ORPOConfig\n\n[[autodoc]] ORPOConfig\n\n## ORPOTrainer\n\n[[autodoc]] ORPOTrainer\n\n## PPOConfig\n\n[[autodoc]] PPOConfig\n\n## PPOTrainer\n\n[[autodoc]] PPOTrainer\n\n## RewardConfig\n\n[[autodoc]] RewardConfig\n\n## RewardTrainer\n\n[[autodoc]] RewardTrainer\n\n## SFTTrainer\n\n[[autodoc]] SFTTrainer\n\n## set_seed\n\n[[autodoc]] set_seed"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 9a8ab62f-a5da-4cba-8d27-797dca8ed56c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/best_of_n", "title": "Best of N sampling: Alternative ways to get better model output without RL based fine-tuning", "tokens": 632, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"9a8ab62f-a5da-4cba-8d27-797dca8ed56c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/best_of_n\", \"title\": \"Best of N sampling: Alternative ways to get better model output without RL based fine-tuning\", \"tokens\": 632, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3107cd7c-607c-416b-8083-5d683865ecb9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/best_of_n\", \"title\": \"Best of N sampling: Alternative ways to get better model output without RL based fine-tuning\", \"tokens\": 632, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"895ecdf57ff22ac34ab7520f0d57da3d6cb1066e23bc7cbb21228475a40fe6ee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2918, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3107cd7c-607c-416b-8083-5d683865ecb9", "doc_id": "3107cd7c-607c-416b-8083-5d683865ecb9", "ref_doc_id": "3107cd7c-607c-416b-8083-5d683865ecb9", "chroma:document": "# Best of N sampling: Alternative ways to get better model output without RL based fine-tuning \n\nWithin the extras module is the `best-of-n` sampler class that serves as an alternative method of generating better model output.\nAs to how it fares against the RL based fine-tuning, please look in the `examples` directory for a comparison example\n\n## Usage\n\nTo get started quickly, instantiate an instance of the class with a model, a length sampler, a tokenizer and a callable that serves as a proxy reward pipeline that outputs reward scores for input queries\n\n```python\n\nfrom transformers import pipeline, AutoTokenizer\nfrom trl import AutoModelForCausalLMWithValueHead\nfrom trl.core import LengthSampler\nfrom trl.extras import BestOfNSampler\n\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained(ref_model_name)\nreward_pipe = pipeline(\"sentiment-analysis\", model=reward_model, device=device)\ntokenizer = AutoTokenizer.from_pretrained(ref_model_name)\ntokenizer.pad_token = tokenizer.eos_token\n\n\n# callable that takes a list of raw text and returns a list of corresponding reward scores\ndef queries_to_scores(list_of_strings):\n  return [output[\"score\"] for output in reward_pipe(list_of_strings)]\n\nbest_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler)\n\n\n```\n\nAnd assuming you have a list/tensor of tokenized queries, you can generate better output by calling the `generate` method\n\n```python\n\nbest_of_n.generate(query_tensors, device=device, **gen_kwargs)\n\n```\nThe default sample size is 4, but you can change it at the time of instance initialization like so\n\n```python\n\nbest_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, sample_size=8)\n\n```\n\nThe default output is the result of taking the top scored output for each query, but you can change it to top 2 and so on by passing the `n_candidates` argument at the time of instance initialization\n\n```python\n\nbest_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, n_candidates=2)\n\n```\n\nThere is the option of setting the generation settings (like `temperature`, `pad_token_id`) at the time of instance creation as opposed to when calling the `generate` method.\nThis is done by passing a `GenerationConfig` from the `transformers` library at the time of initialization\n\n```python\n\nfrom transformers import GenerationConfig\n\ngeneration_config = GenerationConfig(min_length= -1, top_k=0.0, top_p= 1.0, do_sample= True, pad_token_id=tokenizer.eos_token_id)\n\nbest_of_n = BestOfNSampler(model, tokenizer, queries_to_scores, length_sampler=output_length_sampler, generation_config=generation_config)\n\nbest_of_n.generate(query_tensors, device=device)\n\n```\n\nFurthermore, at the time of initialization you can set the seed to control repeatability of the generation process and the number of samples to generate for each query"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 8a72fcd3-74d0-481c-a6ec-10f6a0875be5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/multi_adapter_rl", "title": "Multi Adapter RL (MARL) - a single base model for everything", "tokens": 957, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"8a72fcd3-74d0-481c-a6ec-10f6a0875be5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/multi_adapter_rl\", \"title\": \"Multi Adapter RL (MARL) - a single base model for everything\", \"tokens\": 957, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c14126ac-f163-4113-b1ae-e2225ad52384\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/multi_adapter_rl\", \"title\": \"Multi Adapter RL (MARL) - a single base model for everything\", \"tokens\": 957, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a7b627ed9008d875c904a7e4d585c9eddda5901fba91fe59a62136d0c755b57d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"41a54794-792b-4662-aff0-184281a8667e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e174fa9961aa7a5abad967df46c748536fc309fc83a14f9837a8ef01755ce66b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3114, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "doc_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "ref_doc_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "chroma:document": "# Multi Adapter RL (MARL) - a single base model for everything\n\nHere we present an approach that uses a single base model for the entire PPO algorithm - which includes retrieving the reference logits, computing the active logits and the rewards. This feature is experimental as we did not test the convergence of the approach. We encourage the community to let us know if they potentially face issues.\n\n## Requirements\n\nYou just need to install `peft` and optionally install `bitsandbytes` as well if you want to go for 8bit base models, for more memory efficient finetuning.\n\n## Summary\n\nYou need to address this approach in three stages that we summarize as follows:\n\n1- Train a base model on the target domain (e.g. `imdb` dataset) - this is the Supervised Fine Tuning stage - it can leverage the `SFTTrainer` from TRL.\n2- Train a reward model using `peft`. This is required in order to re-use the adapter during the RL optimisation process (step 3 below). We show an example of leveraging the `RewardTrainer` from TRL in [this example](https://github.com/huggingface/trl/tree/main/examples/scripts/reward_modeling.py)\n3- Fine tune new adapters on the base model using PPO and the reward adapter. (\"0 abstraction RL\")\n\nMake sure to use the same model (i.e. same architecture and same weights) for the stages 2 & 3. \n\n## Quickstart\n\nLet us assume you have trained your reward adapter on `llama-7b` model using `RewardTrainer` and pushed the weights on the hub under `trl-lib/llama-7b-hh-rm-adapter`. \nWhen doing PPO, before passing the model to `PPOTrainer` create your model as follows:\n\n```python\nmodel_name = \"huggyllama/llama-7b\"\nrm_adapter_id = \"trl-lib/llama-7b-hh-rm-adapter\"\n\n# PPO adapter\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(\n    model_name,\n    peft_config=lora_config,\n    reward_adapter=rm_adapter_id,\n)\n\n...\ntrainer = PPOTrainer(\n    model=model,\n    ...\n)\n\n...\n```\nThen inside your PPO training loop, call the `compute_reward_score` method by accessing the `model` attribute from `PPOTrainer`.\n\n```python\nrewards = trainer.model.compute_reward_score(**inputs)\n```\n\n## Advanced usage\n\n### Control on the adapter name \n\nIf you are familiar with the `peft` library, you know that you can use multiple adapters inside the same model. What you can do is train multiple adapters on the same base model to fine-tune on different policies. \nIn this case, you want to be able to control the adapter name you want to activate back, after retrieving the reward. For that, simply pass the appropriate `adapter_name` to `ppo_adapter_name` argument when calling `compute_reward_score`.\n\n```python\nadapter_name_policy_1 = \"policy_1\"\nrewards = trainer.model.compute_reward_score(**inputs, ppo_adapter_name=adapter_name_policy_1)\n...\n```\n\n### Using 4-bit and 8-bit base models\n\nFor more memory efficient fine-tuning, you can load your base model in 8-bit or 4-bit while keeping the adapters in the default precision (float32).\nJust pass the appropriate arguments (i.e."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 41a54794-792b-4662-aff0-184281a8667e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/multi_adapter_rl", "title": "Multi Adapter RL (MARL) - a single base model for everything", "tokens": 957, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"41a54794-792b-4662-aff0-184281a8667e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/multi_adapter_rl\", \"title\": \"Multi Adapter RL (MARL) - a single base model for everything\", \"tokens\": 957, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c14126ac-f163-4113-b1ae-e2225ad52384\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/multi_adapter_rl\", \"title\": \"Multi Adapter RL (MARL) - a single base model for everything\", \"tokens\": 957, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a7b627ed9008d875c904a7e4d585c9eddda5901fba91fe59a62136d0c755b57d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8a72fcd3-74d0-481c-a6ec-10f6a0875be5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/multi_adapter_rl\", \"title\": \"Multi Adapter RL (MARL) - a single base model for everything\", \"tokens\": 957, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"53b3b60fb5d0070c22dc97302b059361929b602c1a9f25aa5e0f0b2dfd1114bf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2158, \"end_char_idx\": 3718, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "doc_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "ref_doc_id": "c14126ac-f163-4113-b1ae-e2225ad52384", "chroma:document": "```python\nrewards = trainer.model.compute_reward_score(**inputs)\n```\n\n## Advanced usage\n\n### Control on the adapter name \n\nIf you are familiar with the `peft` library, you know that you can use multiple adapters inside the same model. What you can do is train multiple adapters on the same base model to fine-tune on different policies. \nIn this case, you want to be able to control the adapter name you want to activate back, after retrieving the reward. For that, simply pass the appropriate `adapter_name` to `ppo_adapter_name` argument when calling `compute_reward_score`.\n\n```python\nadapter_name_policy_1 = \"policy_1\"\nrewards = trainer.model.compute_reward_score(**inputs, ppo_adapter_name=adapter_name_policy_1)\n...\n```\n\n### Using 4-bit and 8-bit base models\n\nFor more memory efficient fine-tuning, you can load your base model in 8-bit or 4-bit while keeping the adapters in the default precision (float32).\nJust pass the appropriate arguments (i.e. `load_in_8bit=True` or `load_in_4bit=True`) to `AutoModelForCausalLMWithValueHead.from_pretrained` as follows (assuming you have installed `bitsandbytes`):\n```python\nmodel_name = \"llama-7b\"\nrm_adapter_id = \"trl-lib/llama-7b-hh-rm-adapter\"\n\n# PPO adapter\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(\n    model_name,\n    peft_config=lora_config,\n    reward_adapter=rm_adapter_id,\n    load_in_8bit=True,\n)\n\n...\ntrainer = PPOTrainer(\n    model=model,\n    ...\n)\n...\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 60136cee-3ab1-4c34-9ad8-ff6ba0b0e7bf
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/logging", "title": "Logging", "tokens": 1666, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"60136cee-3ab1-4c34-9ad8-ff6ba0b0e7bf\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d9953f4-ad38-4840-be8b-8a5bdda14b0b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f4e7478381532136456c5a9f7e6970d1be028e19abdeb0ed19f4bcc5f9151afe\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8afd6035-bde9-4c3c-9408-2935ff640855\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bb910b76713641321500a655f32779c4e104bba3494be04cc81af807ffe4b633\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3067, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "ref_doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "chroma:document": "# Logging\n\nAs reinforcement learning algorithms are historically challenging to debug, it's important to pay careful attention to logging.\nBy default, the TRL [`PPOTrainer`] saves a lot of relevant information to `wandb` or `tensorboard`.\n\nUpon initialization, pass one of these two options to the [`PPOConfig`]:\n```\nconfig = PPOConfig(\n    model_name=args.model_name,\n    log_with=`wandb`, # or `tensorboard`\n)\n```\nIf you want to log with tensorboard, add the kwarg `project_kwargs={\"logging_dir\": PATH_TO_LOGS}` to the PPOConfig.\n\n## PPO Logging\n\nHere's a brief explanation for the logged metrics provided in the data:\n\nKey metrics to monitor. We want to maximize the reward, maintain a low KL divergence, and maximize entropy:\n1. `env/reward_mean`: The average reward obtained from the environment. Alias `ppo/mean_scores`, which is sed to specifically monitor the reward model.\n1. `env/reward_std`: The standard deviation of the reward obtained from the environment. Alias ``ppo/std_scores`, which is sed to specifically monitor the reward model.\n1. `env/reward_dist`: The histogram distribution of the reward obtained from the environment.\n1. `objective/kl`: The mean Kullback-Leibler (KL) divergence between the old and new policies. It measures how much the new policy deviates from the old policy. The KL divergence is used to compute the KL penalty in the objective function.\n1. `objective/kl_dist`: The histogram distribution of the `objective/kl`.\n1. `objective/kl_coef`: The coefficient for Kullback-Leibler (KL) divergence in the objective function. \n1. `ppo/mean_non_score_reward`: The **KL penalty** calculated by `objective/kl * objective/kl_coef` as the total reward for optimization to prevent the new policy from deviating too far from the old policy.\n1. `objective/entropy`: The entropy of the model's policy, calculated by `-logprobs.sum(-1).mean()`. High entropy means the model's actions are more random, which can be beneficial for exploration.\n\nTraining stats:\n1. `ppo/learning_rate`: The learning rate for the PPO algorithm.\n1. `ppo/policy/entropy`: The entropy of the model's policy, calculated by `pd = torch.nn.functional.softmax(logits, dim=-1); entropy = torch.logsumexp(logits, dim=-1) - torch.sum(pd * logits, dim=-1)`. It measures the randomness of the policy.\n1. `ppo/policy/clipfrac`: The fraction of probability ratios (old policy / new policy) that fell outside the clipping range in the PPO objective. This can be used to monitor the optimization process.\n1. `ppo/policy/approxkl`: The approximate KL divergence between the old and new policies, measured by `0.5 * masked_mean((logprobs - old_logprobs) ** 2, mask)`, corresponding to the `k2` estimator in http://joschu.net/blog/kl-approx.html\n1. `ppo/policy/policykl`: Similar to `ppo/policy/approxkl`, but measured by `masked_mean(old_logprobs - logprobs, mask)`, corresponding to the `k1` estimator in http://joschu.net/blog/kl-approx.html\n1. `ppo/policy/ratio`:  The histogram distribution of the ratio between the new and old policies, used to compute the PPO objective.\n1."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 8afd6035-bde9-4c3c-9408-2935ff640855
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/logging", "title": "Logging", "tokens": 1666, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"8afd6035-bde9-4c3c-9408-2935ff640855\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d9953f4-ad38-4840-be8b-8a5bdda14b0b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f4e7478381532136456c5a9f7e6970d1be028e19abdeb0ed19f4bcc5f9151afe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"60136cee-3ab1-4c34-9ad8-ff6ba0b0e7bf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"617030113c854157218d0577d7d1874153306da8d2d1a8127fb56189005aef3a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1e89b0b3-755d-4285-bac4-9d8cd67a91bd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4b02e3a448a8026629e1a03afbd5d608ef6b86919d33528cef4277c855d32fdb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1564, \"end_char_idx\": 4609, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "ref_doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "chroma:document": "1. `ppo/mean_non_score_reward`: The **KL penalty** calculated by `objective/kl * objective/kl_coef` as the total reward for optimization to prevent the new policy from deviating too far from the old policy.\n1. `objective/entropy`: The entropy of the model's policy, calculated by `-logprobs.sum(-1).mean()`. High entropy means the model's actions are more random, which can be beneficial for exploration.\n\nTraining stats:\n1. `ppo/learning_rate`: The learning rate for the PPO algorithm.\n1. `ppo/policy/entropy`: The entropy of the model's policy, calculated by `pd = torch.nn.functional.softmax(logits, dim=-1); entropy = torch.logsumexp(logits, dim=-1) - torch.sum(pd * logits, dim=-1)`. It measures the randomness of the policy.\n1. `ppo/policy/clipfrac`: The fraction of probability ratios (old policy / new policy) that fell outside the clipping range in the PPO objective. This can be used to monitor the optimization process.\n1. `ppo/policy/approxkl`: The approximate KL divergence between the old and new policies, measured by `0.5 * masked_mean((logprobs - old_logprobs) ** 2, mask)`, corresponding to the `k2` estimator in http://joschu.net/blog/kl-approx.html\n1. `ppo/policy/policykl`: Similar to `ppo/policy/approxkl`, but measured by `masked_mean(old_logprobs - logprobs, mask)`, corresponding to the `k1` estimator in http://joschu.net/blog/kl-approx.html\n1. `ppo/policy/ratio`:  The histogram distribution of the ratio between the new and old policies, used to compute the PPO objective.\n1. `ppo/policy/advantages_mean`: The average of the GAE (Generalized Advantage Estimation) advantage estimates. The advantage function measures how much better an action is compared to the average action at a state.\n1. `ppo/policy/advantages`: The histogram distribution of `ppo/policy/advantages_mean`.\n1. `ppo/returns/mean`: The mean of the TD(\u03bb) returns, calculated by `returns = advantage + values`, another indicator of model performance. See https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/ for more details.\n1. `ppo/returns/var`: The variance of the TD(\u03bb) returns, calculated by `returns = advantage + values`, another indicator of model performance.\n1. `ppo/val/mean`: The mean of the values, used to monitor the value function's performance.\n1. `ppo/val/var` : The variance of the values, used to monitor the value function's performance.\n1. `ppo/val/var_explained`: The explained variance for the value function, used to monitor the value function's performance.\n1. `ppo/val/clipfrac`: The fraction of the value function's predicted values that are clipped.\n1. `ppo/val/vpred`: The predicted values from the value function.\n1. `ppo/val/error`: The mean squared error between the `ppo/val/vpred` and returns, used to monitor the value function's performance.\n1. `ppo/loss/policy`: The policy loss for the Proximal Policy Optimization (PPO) algorithm.\n1. `ppo/loss/value`: The loss for the value function in the PPO algorithm. This value quantifies how well the function estimates the expected future rewards.\n1."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 1e89b0b3-755d-4285-bac4-9d8cd67a91bd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/logging", "title": "Logging", "tokens": 1666, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"1e89b0b3-755d-4285-bac4-9d8cd67a91bd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d9953f4-ad38-4840-be8b-8a5bdda14b0b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f4e7478381532136456c5a9f7e6970d1be028e19abdeb0ed19f4bcc5f9151afe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8afd6035-bde9-4c3c-9408-2935ff640855\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"fafa2f337af8a0c29f3ba359d1e82725e95647197b9abae775c4c18a61df7355\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7c75c4e9-df2f-411d-98f4-ec74b1b3fca8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5eb96a8d9c216c92988f1cfae39ac3921cb62be8af0fd48fe4ce9b45cabb8030\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3065, \"end_char_idx\": 5592, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "ref_doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "chroma:document": "1. `ppo/policy/advantages_mean`: The average of the GAE (Generalized Advantage Estimation) advantage estimates. The advantage function measures how much better an action is compared to the average action at a state.\n1. `ppo/policy/advantages`: The histogram distribution of `ppo/policy/advantages_mean`.\n1. `ppo/returns/mean`: The mean of the TD(\u03bb) returns, calculated by `returns = advantage + values`, another indicator of model performance. See https://iclr-blog-track.github.io/2022/03/25/ppo-implementation-details/ for more details.\n1. `ppo/returns/var`: The variance of the TD(\u03bb) returns, calculated by `returns = advantage + values`, another indicator of model performance.\n1. `ppo/val/mean`: The mean of the values, used to monitor the value function's performance.\n1. `ppo/val/var` : The variance of the values, used to monitor the value function's performance.\n1. `ppo/val/var_explained`: The explained variance for the value function, used to monitor the value function's performance.\n1. `ppo/val/clipfrac`: The fraction of the value function's predicted values that are clipped.\n1. `ppo/val/vpred`: The predicted values from the value function.\n1. `ppo/val/error`: The mean squared error between the `ppo/val/vpred` and returns, used to monitor the value function's performance.\n1. `ppo/loss/policy`: The policy loss for the Proximal Policy Optimization (PPO) algorithm.\n1. `ppo/loss/value`: The loss for the value function in the PPO algorithm. This value quantifies how well the function estimates the expected future rewards.\n1. `ppo/loss/total`: The total loss for the PPO algorithm. It is the sum of the policy loss and the value function loss.\n\n\nStats on queries, responses, and logprobs:\n1. `tokens/queries_len_mean`: The average length of the queries tokens.\n1. `tokens/queries_len_std`: The standard deviation of the length of the queries tokens.\n1. `tokens/queries_dist`: The histogram distribution of the length of the queries tokens.\n1. `tokens/responses_len_mean`: The average length of the responses tokens.\n1. `tokens/responses_len_std`: The standard deviation of the length of the responses tokens.\n1. `tokens/responses_dist`: The histogram distribution of the length of the responses tokens. (Costa: inconsistent naming, should be `tokens/responses_len_dist`)\n1. `objective/logprobs`: The histogram distribution of the log probabilities of the actions taken by the model.\n1. `objective/ref_logprobs`: The histogram distribution of the log probabilities of the actions taken by the reference model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 7c75c4e9-df2f-411d-98f4-ec74b1b3fca8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/logging", "title": "Logging", "tokens": 1666, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"7c75c4e9-df2f-411d-98f4-ec74b1b3fca8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d9953f4-ad38-4840-be8b-8a5bdda14b0b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"f4e7478381532136456c5a9f7e6970d1be028e19abdeb0ed19f4bcc5f9151afe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1e89b0b3-755d-4285-bac4-9d8cd67a91bd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/logging\", \"title\": \"Logging\", \"tokens\": 1666, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"525b1536b1b6bead41e9ac131c57aae653fda4592a52042cb8878ceebfed0c95\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3937, \"end_char_idx\": 7033, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "ref_doc_id": "3d9953f4-ad38-4840-be8b-8a5bdda14b0b", "chroma:document": "1. `ppo/val/var_explained`: The explained variance for the value function, used to monitor the value function's performance.\n1. `ppo/val/clipfrac`: The fraction of the value function's predicted values that are clipped.\n1. `ppo/val/vpred`: The predicted values from the value function.\n1. `ppo/val/error`: The mean squared error between the `ppo/val/vpred` and returns, used to monitor the value function's performance.\n1. `ppo/loss/policy`: The policy loss for the Proximal Policy Optimization (PPO) algorithm.\n1. `ppo/loss/value`: The loss for the value function in the PPO algorithm. This value quantifies how well the function estimates the expected future rewards.\n1. `ppo/loss/total`: The total loss for the PPO algorithm. It is the sum of the policy loss and the value function loss.\n\n\nStats on queries, responses, and logprobs:\n1. `tokens/queries_len_mean`: The average length of the queries tokens.\n1. `tokens/queries_len_std`: The standard deviation of the length of the queries tokens.\n1. `tokens/queries_dist`: The histogram distribution of the length of the queries tokens.\n1. `tokens/responses_len_mean`: The average length of the responses tokens.\n1. `tokens/responses_len_std`: The standard deviation of the length of the responses tokens.\n1. `tokens/responses_dist`: The histogram distribution of the length of the responses tokens. (Costa: inconsistent naming, should be `tokens/responses_len_dist`)\n1. `objective/logprobs`: The histogram distribution of the log probabilities of the actions taken by the model.\n1. `objective/ref_logprobs`: The histogram distribution of the log probabilities of the actions taken by the reference model.\n\n\n\n### Crucial values\nDuring training, many values are logged, here are the most important ones:\n\n1. `env/reward_mean`,`env/reward_std`, `env/reward_dist`: the properties of the reward distribution from the \"environment\" /  reward model\n1. `ppo/mean_non_score_reward`: The mean negated KL penalty during training (shows the delta between the reference model and the new policy over the batch in the step)\n\nHere are some parameters that are useful to monitor for stability (when these diverge or collapse to 0, try tuning variables):\n\n1. `ppo/loss/value`: it will spike / NaN when not going well.\n1. `ppo/policy/ratio`: `ratio` being 1 is a baseline value, meaning that the probability of sampling a token is the same under the new and old policy. If the ratio is too high like 200, it means the probability of sampling a token is 200 times higher under the new policy than the old policy. This is a sign that the new policy is too different from the old policy, which will likely cause overoptimization and collapse training later on.\n1. `ppo/policy/clipfrac` and `ppo/policy/approxkl`: if `ratio` is too high, the `ratio` is going to get clipped, resulting in high `clipfrac` and high `approxkl` as well.\n1. `objective/kl`: it should stay positive so that the policy is not too far away from the reference policy.\n1. `objective/kl_coef`: The target coefficient with [`AdaptiveKLController`]. Often increases before numerical instabilities."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5fc4a359-59cc-4390-881e-b64e61b6aeb5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/kto_trainer", "title": "KTO Trainer", "tokens": 970, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"5fc4a359-59cc-4390-881e-b64e61b6aeb5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/kto_trainer\", \"title\": \"KTO Trainer\", \"tokens\": 970, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1313028e-be5c-4d19-bf43-f36d54e284ff\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/kto_trainer\", \"title\": \"KTO Trainer\", \"tokens\": 970, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"dddcad02280f1d2bbcbdae915ebf04604be7cb5b6cd81051138f14b7e05bf056\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"be8250ec-2445-4b73-b7cf-f23022a64935\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"738ce6f70f50a9c7fdcb6b6166503cf44b84e4e38a57706801607893bfbfcc78\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2019, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "doc_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "ref_doc_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "chroma:document": "# KTO Trainer\n\nTRL supports the Kahneman-Tversky Optimization (KTO) Trainer for aligning language models with binary feedback data (e.g., upvote/downvote), as described in the [paper](https://huggingface.co/papers/2402.01306) by Kawin Ethayarajh, Winnie Xu, Niklas Muennighoff, Dan Jurafsky, and Douwe Kiela.\nFor a full example have a look at  [`examples/scripts/kto.py`].\n\nDepending on how good your base model is, you may or may not need to do SFT before KTO.\nThis is different from standard RLHF and DPO, which always require SFT.\n\n## Expected dataset format\n\nThe KTO trainer expects a very specific format for the dataset as it does not require pairwise preferences. Since the model will be trained to directly optimize examples that consist of a prompt, model completion, and a label to indicate whether the completion is \"good\" or \"bad\", we expect a dataset with the following columns:\n\n- `prompt`\n- `completion`\n- `label`\n\nfor example:\n\n```\nkto_dataset_dict = {\n    \"prompt\": [\n        \"Hey, hello\",\n        \"How are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"completion\": [\n        \"hi nice to meet you\",\n        \"leave me alone\",\n        \"I don't have a name\",\n        \"My name is Mary\",\n        \"Python\",\n        \"C++\",\n        \"Java\",\n    ],\n    \"label\": [\n        True,\n        False,\n        False,\n        True,\n        True,\n        False,\n        False,\n    ],\n}\n```\n\nwhere the `prompt` contains the context inputs, `completion` contains the corresponding responses and `label` contains the corresponding flag that indicates if the generated completion is desired (`True`) or undesired (`False`).\nA prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays. It is required that the dataset contains at least one desirable and one undesirable completion."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: be8250ec-2445-4b73-b7cf-f23022a64935
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/kto_trainer", "title": "KTO Trainer", "tokens": 970, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"be8250ec-2445-4b73-b7cf-f23022a64935\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/kto_trainer\", \"title\": \"KTO Trainer\", \"tokens\": 970, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1313028e-be5c-4d19-bf43-f36d54e284ff\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/kto_trainer\", \"title\": \"KTO Trainer\", \"tokens\": 970, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"dddcad02280f1d2bbcbdae915ebf04604be7cb5b6cd81051138f14b7e05bf056\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5fc4a359-59cc-4390-881e-b64e61b6aeb5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/kto_trainer\", \"title\": \"KTO Trainer\", \"tokens\": 970, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"e433693c29be5202ae9694b76c661d85c279b95a235ee485ac19de7c6e80c88d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2022, \"end_char_idx\": 4078, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "doc_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "ref_doc_id": "1313028e-be5c-4d19-bf43-f36d54e284ff", "chroma:document": "## Expected model format\nThe KTO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `KTOTrainer`\n\nFor a detailed example have a look at the `examples/scripts/kto.py` script. At a high level we need to initialize the `KTOTrainer` with a `model` we wish to train and a reference `ref_model` which we will use to calculate the implicit rewards of the preferred and rejected response. \n\nThe `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above. Note that the `model` and `ref_model` need to have the same architecture (ie decoder only or encoder-decoder).\n\nThe `desirable_weight` and `undesirable_weight` refer to the weights placed on the losses for desirable/positive and undesirable/negative examples.\nBy default, they are both 1. However, if you have more of one or the other, then you should upweight the less common type such that the ratio of (`desirable_weight` * number of positives) to (`undesirable_weight` * number of negatives) is in the range 1:1 to 4:3.\n\n```py\ntraining_args = KTOConfig(\n    beta=0.1,\n    desirable_weight=1.0,\n    undesirable_weight=1.0,\n)\n\nkto_trainer = KTOTrainer(\n    model,\n    ref_model,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\nkto_trainer.train()\n```\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.  \n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001).\n\n## KTOTrainer\n\n[[autodoc]] KTOTrainer\n\n## KTOConfig\n\n[[autodoc]] KTOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2a660d4d-9dd8-4621-bc3b-25531e227871
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/alignprop_trainer", "title": "Aligning Text-to-Image Diffusion Models with Reward Backpropagation", "tokens": 1189, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2a660d4d-9dd8-4621-bc3b-25531e227871\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/alignprop_trainer\", \"title\": \"Aligning Text-to-Image Diffusion Models with Reward Backpropagation\", \"tokens\": 1189, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1d57c232-95bd-4519-aeba-c26fbc873ba2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/alignprop_trainer\", \"title\": \"Aligning Text-to-Image Diffusion Models with Reward Backpropagation\", \"tokens\": 1189, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4e06c25df8950d7ab028a71bdf6261c3d9fead6dadb00dc0ddcd78c20671c06b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3ee149cb-4e80-4012-ae2e-be66070817df\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d6b15aeb98b4d6b00e167bcc35ce931cc3c994969d59e4dedf890789a67dfabc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3430, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "doc_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "ref_doc_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "chroma:document": "# Aligning Text-to-Image Diffusion Models with Reward Backpropagation\n\n## The why\n\nIf your reward function is differentiable, directly backpropagating gradients from the reward models to the diffusion model is significantly more sample and compute efficient (25x) than doing policy gradient algorithm like DDPO.\nAlignProp does full backpropagation through time, which allows updating the earlier steps of denoising via reward backpropagation.\n\n<div style=\"text-align: center\"><img src=\"https://align-prop.github.io/reward_tuning.png\"/></div>\n\n\n## Getting started with `examples/scripts/alignprop.py`\n\nThe `alignprop.py` script is a working example of using the `AlignProp` trainer to finetune a Stable Diffusion model. This example explicitly configures a small subset of the overall parameters associated with the config object (`AlignPropConfig`).\n\n**Note:** one A100 GPU is recommended to get this running. For lower memory setting, consider setting truncated_backprop_rand to False. With default settings this will do truncated backpropagation with K=1.\n\nAlmost every configuration parameter has a default. There is only one commandline flag argument that is required of the user to get things up and running. The user is expected to have a [huggingface user access token](https://huggingface.co/docs/hub/security-tokens) that will be used to upload the model post finetuning to HuggingFace hub. The following bash command is to be entered to get things running\n\n```batch\npython alignprop.py --hf_user_access_token <token>\n```\n\nTo obtain the documentation of `stable_diffusion_tuning.py`, please run `python stable_diffusion_tuning.py --help`\n\nThe following are things to keep in mind (The code checks this for you as well) in general while configuring the trainer (beyond the use case of using the example script)\n\n- The configurable randomized truncation range (`--alignprop_config.truncated_rand_backprop_minmax=(0,50)`) the first number should be equal and greater to 0, while the second number should equal or less to the number of diffusion timesteps (sample_num_steps)\n- The configurable truncation backprop absolute step (`--alignprop_config.truncated_backprop_timestep=49`) the number should be less than the number of diffusion timesteps (sample_num_steps), it only matters when truncated_backprop_rand is set to False\n\n## Setting up the image logging hook function\n\nExpect the function to be given a dictionary with keys\n```python\n['image', 'prompt', 'prompt_metadata', 'rewards']\n\n```\nand `image`, `prompt`, `prompt_metadata`, `rewards`are batched.\nYou are free to log however you want the use of `wandb` or `tensorboard` is recommended.\n\n### Key terms\n\n- `rewards` : The rewards/score is a numerical associated with the generated image and is key to steering the RL process\n- `prompt` : The prompt is the text that is used to generate the image\n- `prompt_metadata` : The prompt metadata is the metadata associated with the prompt. A situation where this will not be empty is when the reward model comprises of a [`FLAVA`](https://huggingface.co/docs/transformers/model_doc/flava) setup where questions and ground answers (linked to the generated image) are expected with the generated image (See here: https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45)\n- `image` : The image generated by the Stable Diffusion model\n\nExample code for logging sampled images with `wandb` is given below."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3ee149cb-4e80-4012-ae2e-be66070817df
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/alignprop_trainer", "title": "Aligning Text-to-Image Diffusion Models with Reward Backpropagation", "tokens": 1189, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3ee149cb-4e80-4012-ae2e-be66070817df\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/alignprop_trainer\", \"title\": \"Aligning Text-to-Image Diffusion Models with Reward Backpropagation\", \"tokens\": 1189, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1d57c232-95bd-4519-aeba-c26fbc873ba2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/alignprop_trainer\", \"title\": \"Aligning Text-to-Image Diffusion Models with Reward Backpropagation\", \"tokens\": 1189, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4e06c25df8950d7ab028a71bdf6261c3d9fead6dadb00dc0ddcd78c20671c06b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2a660d4d-9dd8-4621-bc3b-25531e227871\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/alignprop_trainer\", \"title\": \"Aligning Text-to-Image Diffusion Models with Reward Backpropagation\", \"tokens\": 1189, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"be5dd7fd262c8c734dea2aafb621d757569f6718b857970e9e919f143c9b681f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2565, \"end_char_idx\": 5021, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "doc_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "ref_doc_id": "1d57c232-95bd-4519-aeba-c26fbc873ba2", "chroma:document": "You are free to log however you want the use of `wandb` or `tensorboard` is recommended.\n\n### Key terms\n\n- `rewards` : The rewards/score is a numerical associated with the generated image and is key to steering the RL process\n- `prompt` : The prompt is the text that is used to generate the image\n- `prompt_metadata` : The prompt metadata is the metadata associated with the prompt. A situation where this will not be empty is when the reward model comprises of a [`FLAVA`](https://huggingface.co/docs/transformers/model_doc/flava) setup where questions and ground answers (linked to the generated image) are expected with the generated image (See here: https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45)\n- `image` : The image generated by the Stable Diffusion model\n\nExample code for logging sampled images with `wandb` is given below.\n\n```python\n# for logging these images to wandb\n\ndef image_outputs_hook(image_data, global_step, accelerate_logger):\n    # For the sake of this example, we only care about the last batch\n    # hence we extract the last element of the list\n    result = {}\n    images, prompts, rewards = [image_data['images'],image_data['prompts'],image_data['rewards']]\n    for i, image in enumerate(images):\n        pil = Image.fromarray(\n            (image.cpu().numpy().transpose(1, 2, 0) * 255).astype(np.uint8)\n        )\n        pil = pil.resize((256, 256))\n        result[f\"{prompts[i]:.25} | {rewards[i]:.2f}\"] = [pil]\n    accelerate_logger.log_images(\n        result,\n        step=global_step,\n    )\n\n```\n\n### Using the finetuned model\n\nAssuming you've done with all the epochs and have pushed up your model to the hub, you can use the finetuned model as follows\n\n```python\nfrom diffusers import StableDiffusionPipeline\npipeline = StableDiffusionPipeline.from_pretrained(\"runwayml/stable-diffusion-v1-5\")\npipeline.to(\"cuda\")\n\npipeline.load_lora_weights('mihirpd/alignprop-trl-aesthetics')\n\nprompts = [\"squirrel\", \"crab\", \"starfish\", \"whale\",\"sponge\", \"plankton\"]\nresults = pipeline(prompts)\n\nfor prompt, image in zip(prompts,results.images):\n    image.save(f\"dump/{prompt}.png\")\n```\n\n## Credits\n\nThis work is heavily influenced by the repo [here](https://github.com/mihirp1998/AlignProp/) and the associated paper [Aligning Text-to-Image Diffusion Models with Reward Backpropagation\n by Mihir Prabhudesai, Anirudh Goyal, Deepak Pathak, Katerina Fragkiadaki](https://huggingface.co/papers/2310.03739)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: af821a28-8ff8-4b3d-9226-15aab889c582
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sentiment_tuning", "title": "Sentiment Tuning Examples", "tokens": 1601, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"af821a28-8ff8-4b3d-9226-15aab889c582\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"068713a8-03d9-4ee0-9160-1eddea0b396e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"79b3ef31caa859bcf817dbfbf8ebf948e5aed5c4469fc35486d3971cb19fea23\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"179b4a0c-d18f-4dc2-bd4e-e0757808623b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f003098101663a6c7944f3f673c12938cbd6392231914a88f8ee6d93e63dacab\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3065, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "ref_doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "chroma:document": "# Sentiment Tuning Examples\n\nThe notebooks and scripts in this examples show how to fine-tune a model with a sentiment classifier (such as `lvwerra/distilbert-imdb`).\n\nHere's an overview of the notebooks and scripts in the [trl repository](https://github.com/huggingface/trl/tree/main/examples):\n\n\n\n| File                                                                                           | Description                                                                                                              |\n|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|\n| [`examples/scripts/ppo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo.py)  [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sentiment/notebooks/gpt2-sentiment.ipynb) | This script shows how to use the `PPOTrainer` to fine-tune a sentiment analysis model using IMDB dataset                 |\n| [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-sentiment.ipynb)              | This notebook demonstrates how to reproduce the GPT2 imdb sentiment tuning example on a jupyter notebook.                |\n| [`examples/notebooks/gpt2-control.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-control.ipynb)   [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/trl/blob/main/examples/sentiment/notebooks/gpt2-sentiment-control.ipynb)                | This notebook demonstrates how to reproduce the GPT2 sentiment control example on a jupyter notebook.    \n\n\n\n## Usage\n\n```bash\n# 1. run directly\npython examples/scripts/ppo.py\n# 2. run via `accelerate` (recommended), enabling more features (e.g., multiple GPUs, deepspeed)\naccelerate config # will prompt you to define the training configuration\naccelerate launch examples/scripts/ppo.py # launches training\n# 3. get help text and documentation\npython examples/scripts/ppo.py --help\n# 4. configure logging with wandb and, say, mini_batch_size=1 and gradient_accumulation_steps=16\npython examples/scripts/ppo.py --log_with wandb --mini_batch_size 1 --gradient_accumulation_steps 16\n```\n\nNote: if you don't want to log with `wandb` remove `log_with=\"wandb\"` in the scripts/notebooks. You can also replace it with your favourite experiment tracker that's [supported by `accelerate`](https://huggingface.co/docs/accelerate/usage_guides/tracking).\n\n\n## Few notes on multi-GPU \n\nTo run in multi-GPU setup with DDP (distributed Data Parallel) change the `device_map` value to `device_map={\"\": Accelerator().process_index}` and make sure to run your script with `accelerate launch yourscript.py`. If you want to apply naive pipeline parallelism you can use `device_map=\"auto\"`."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 179b4a0c-d18f-4dc2-bd4e-e0757808623b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sentiment_tuning", "title": "Sentiment Tuning Examples", "tokens": 1601, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"179b4a0c-d18f-4dc2-bd4e-e0757808623b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"068713a8-03d9-4ee0-9160-1eddea0b396e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"79b3ef31caa859bcf817dbfbf8ebf948e5aed5c4469fc35486d3971cb19fea23\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"af821a28-8ff8-4b3d-9226-15aab889c582\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3e370eb195817283eb7f1a6f4e2261ffcdea933411475325b7c98763cb7c5371\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"60852c40-ee68-4ffc-8fde-6d3ca4432fc1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9cbe88825ff1a880f33221eeafb905e727666dd9a02e23ce4a8d0736580e0596\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1905, \"end_char_idx\": 4296, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "ref_doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "chroma:document": "## Usage\n\n```bash\n# 1. run directly\npython examples/scripts/ppo.py\n# 2. run via `accelerate` (recommended), enabling more features (e.g., multiple GPUs, deepspeed)\naccelerate config # will prompt you to define the training configuration\naccelerate launch examples/scripts/ppo.py # launches training\n# 3. get help text and documentation\npython examples/scripts/ppo.py --help\n# 4. configure logging with wandb and, say, mini_batch_size=1 and gradient_accumulation_steps=16\npython examples/scripts/ppo.py --log_with wandb --mini_batch_size 1 --gradient_accumulation_steps 16\n```\n\nNote: if you don't want to log with `wandb` remove `log_with=\"wandb\"` in the scripts/notebooks. You can also replace it with your favourite experiment tracker that's [supported by `accelerate`](https://huggingface.co/docs/accelerate/usage_guides/tracking).\n\n\n## Few notes on multi-GPU \n\nTo run in multi-GPU setup with DDP (distributed Data Parallel) change the `device_map` value to `device_map={\"\": Accelerator().process_index}` and make sure to run your script with `accelerate launch yourscript.py`. If you want to apply naive pipeline parallelism you can use `device_map=\"auto\"`.\n\n\n## Benchmarks\n\nBelow are some benchmark results for `examples/scripts/ppo.py`. To reproduce locally, please check out the `--command` arguments below.\n\n```bash\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/sentiment.png)\n\n\n\n## With and without gradient accumulation\n\n```bash\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_step_grad_accu --mini_batch_size 1 --gradient_accumulation_steps 128 --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/gradient_accu.png)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 60852c40-ee68-4ffc-8fde-6d3ca4432fc1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/sentiment_tuning", "title": "Sentiment Tuning Examples", "tokens": 1601, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"60852c40-ee68-4ffc-8fde-6d3ca4432fc1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"068713a8-03d9-4ee0-9160-1eddea0b396e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"79b3ef31caa859bcf817dbfbf8ebf948e5aed5c4469fc35486d3971cb19fea23\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"179b4a0c-d18f-4dc2-bd4e-e0757808623b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/sentiment_tuning\", \"title\": \"Sentiment Tuning Examples\", \"tokens\": 1601, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"da21ba8bf295d3f7ac50b0689621b78b901bd76b57373025b7884cfd761db295\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3068, \"end_char_idx\": 6218, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "ref_doc_id": "068713a8-03d9-4ee0-9160-1eddea0b396e", "chroma:document": "## Benchmarks\n\nBelow are some benchmark results for `examples/scripts/ppo.py`. To reproduce locally, please check out the `--command` arguments below.\n\n```bash\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/sentiment.png)\n\n\n\n## With and without gradient accumulation\n\n```bash\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_step_grad_accu --mini_batch_size 1 --gradient_accumulation_steps 128 --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/gradient_accu.png)\n\n\n## Comparing different models (gpt2, gpt2-xl, falcon, llama2)\n\n```bash\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_gpt2 --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_gpt2xl_grad_accu --model_name gpt2-xl --mini_batch_size 16 --gradient_accumulation_steps 8 --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_falcon_rw_1b --model_name tiiuae/falcon-rw-1b --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/different_models.png)\n\n## With and without PEFT\n\n```\npython benchmark/benchmark.py \\\n    --command \"python examples/scripts/ppo.py --exp_name sentiment_tuning_peft --use_peft --log_with wandb\" \\\n    --num-seeds 5 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-nodes 1 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 12 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/benchmark/v0.4.7-55-g110e672/peft.png)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: f4faa8b7-d133-4c68-8ec6-97fe4a3028d5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/quickstart", "title": "Quickstart", "tokens": 902, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"f4faa8b7-d133-4c68-8ec6-97fe4a3028d5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/quickstart\", \"title\": \"Quickstart\", \"tokens\": 902, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"405128a5-d00d-4adc-92ec-60fe87f9a3c0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/quickstart\", \"title\": \"Quickstart\", \"tokens\": 902, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"8bfbd730047ec01b65ac7e0f6fadff824f713bf9cff56ce01daf95fd4d5fc15b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f2183629-2d1f-4a09-b744-b93a5ae205b0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2969d01d4915ee549e7b3c50dcd22f3ed38801b803988640fa435bda14a574c2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1438, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "doc_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "ref_doc_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "chroma:document": "# Quickstart\n\n## How does it work?\n\nFine-tuning a language model via PPO consists of roughly three steps:\n\n1. **Rollout**: The language model generates a response or continuation based on a query which could be the start of a sentence.\n2. **Evaluation**: The query and response are evaluated with a function, model, human feedback, or some combination of them. The important thing is that this process should yield a scalar value for each query/response pair. The optimization will aim at maximizing this value.\n3. **Optimization**: This is the most complex part. In the optimisation step the query/response pairs are used to calculate the log-probabilities of the tokens in the sequences. This is done with the model that is trained and a reference model, which is usually the pre-trained model before fine-tuning. The KL-divergence between the two outputs is used as an additional reward signal to make sure the generated responses don't deviate too far from the reference language model. The active language model is then trained with PPO.\n\nThe full process is illustrated in the following figure:\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl_overview.png\"/>\n\n## Minimal example\n\nThe following code illustrates the steps above. \n\n```python\n# 0. imports\nimport torch\nfrom transformers import GPT2Tokenizer\n\nfrom trl import AutoModelForCausalLMWithValueHead, PPOConfig, PPOTrainer"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: f2183629-2d1f-4a09-b744-b93a5ae205b0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/quickstart", "title": "Quickstart", "tokens": 902, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"f2183629-2d1f-4a09-b744-b93a5ae205b0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/quickstart\", \"title\": \"Quickstart\", \"tokens\": 902, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"405128a5-d00d-4adc-92ec-60fe87f9a3c0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/quickstart\", \"title\": \"Quickstart\", \"tokens\": 902, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"8bfbd730047ec01b65ac7e0f6fadff824f713bf9cff56ce01daf95fd4d5fc15b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f4faa8b7-d133-4c68-8ec6-97fe4a3028d5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/quickstart\", \"title\": \"Quickstart\", \"tokens\": 902, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0745f8e1a2bf1a924bee522813462572f800ef07ceb588027ded3debd371aefd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3690, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "doc_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "ref_doc_id": "405128a5-d00d-4adc-92ec-60fe87f9a3c0", "chroma:document": "# Quickstart\n\n## How does it work?\n\nFine-tuning a language model via PPO consists of roughly three steps:\n\n1. **Rollout**: The language model generates a response or continuation based on a query which could be the start of a sentence.\n2. **Evaluation**: The query and response are evaluated with a function, model, human feedback, or some combination of them. The important thing is that this process should yield a scalar value for each query/response pair. The optimization will aim at maximizing this value.\n3. **Optimization**: This is the most complex part. In the optimisation step the query/response pairs are used to calculate the log-probabilities of the tokens in the sequences. This is done with the model that is trained and a reference model, which is usually the pre-trained model before fine-tuning. The KL-divergence between the two outputs is used as an additional reward signal to make sure the generated responses don't deviate too far from the reference language model. The active language model is then trained with PPO.\n\nThe full process is illustrated in the following figure:\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/trl_overview.png\"/>\n\n## Minimal example\n\nThe following code illustrates the steps above. \n\n```python\n# 0. imports\nimport torch\nfrom transformers import GPT2Tokenizer\n\nfrom trl import AutoModelForCausalLMWithValueHead, PPOConfig, PPOTrainer\n\n\n# 1. load a pretrained model\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(\"gpt2\")\nref_model = AutoModelForCausalLMWithValueHead.from_pretrained(\"gpt2\")\ntokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\ntokenizer.pad_token = tokenizer.eos_token\n\n# 2. initialize trainer\nppo_config = {\"mini_batch_size\": 1, \"batch_size\": 1}\nconfig = PPOConfig(**ppo_config)\nppo_trainer = PPOTrainer(config, model, ref_model, tokenizer)\n\n# 3. encode a query\nquery_txt = \"This morning I went to the \"\nquery_tensor = tokenizer.encode(query_txt, return_tensors=\"pt\").to(model.pretrained_model.device)\n\n# 4. generate model response\ngeneration_kwargs = {\n    \"min_length\": -1,\n    \"top_k\": 0.0,\n    \"top_p\": 1.0,\n    \"do_sample\": True,\n    \"pad_token_id\": tokenizer.eos_token_id,\n    \"max_new_tokens\": 20,\n}\nresponse_tensor = ppo_trainer.generate([item for item in query_tensor], return_prompt=False, **generation_kwargs)\nresponse_txt = tokenizer.decode(response_tensor[0])\n\n# 5. define a reward for response\n# (this could be any reward such as human feedback or output from another model)\nreward = [torch.tensor(1.0, device=model.pretrained_model.device)]\n\n# 6. train model with ppo\ntrain_stats = ppo_trainer.step([query_tensor[0]], [response_tensor[0]], reward)\n```\n\nIn general, you would run steps 3-6 in a for-loop and run it on many diverse queries. You can find more realistic examples in the examples section. \n\n## How to use a trained model\n\nAfter training a `AutoModelForCausalLMWithValueHead`, you can directly use the model in `transformers`.\n```python\n\n# .. Let's assume we have a trained model using `PPOTrainer` and `AutoModelForCausalLMWithValueHead`\n\n# push the model on the Hub\nmodel.push_to_hub(\"my-fine-tuned-model-ppo\")\n\n# or save it locally\nmodel.save_pretrained(\"my-fine-tuned-model-ppo\")\n\n# load the model from the Hub\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"my-fine-tuned-model-ppo\")\n```\n\nYou can also load your model with `AutoModelForCausalLMWithValueHead` if you want to use the value head, for example to continue training.\n\n```python\nfrom trl.model import AutoModelForCausalLMWithValueHead\n\nmodel = AutoModelForCausalLMWithValueHead.from_pretrained(\"my-fine-tuned-model-ppo\")\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d1fb3764-3e95-458d-8619-e8ed5596061c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/text_environments", "title": "Text Environments", "tokens": 2275, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d1fb3764-3e95-458d-8619-e8ed5596061c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1bd6dc94-02bd-4f50-b752-0cbe7044b926\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cba40e36c2453aef7be446350c9a4c783340bf7cb3c68911d89d32228a01679\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d2fe3676-ee89-4ad4-a612-728e4afddaf8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"512df27de01bdf0c8fe468c0ef6659205df4695f2885cafec9562f39f057ec80\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3321, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "ref_doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "chroma:document": "# Text Environments\n\nText environments provide a learning ground for language agents. It allows a language model to use tools to accomplish a task such as using a Python interpreter to answer math questions or using a search index for trivia questions. Having access to tools allows language models to solve tasks that would be very hard for the models itself but can be trivial for the appropriate tools. A good example is arithmetics of large numbers that become a simple copy-paste task once you have access to a calculator.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/textenv.png\">\n</div>\n\nLet's dive into how text environments work and start with tools!\n\n## Tools\n\nOne of the core building blocks of text environments are tools that the model can use to solve tasks. In general tools can be any Python function that takes a string as input and returns string. The `TextEnvironment` offers two options for tools: either go with predefined tools from `transformers.Tool` or define your own function or class with `__call__` method. Let's have a look at both!\n\n### `transformers.Tool`\n\nText environments fully support tools of the class `transformers.Tool`. The advantage of building tools in that framework is that they can easily be shared \n\n```Python\nfrom transformers import load_tool\n\n# simple calculator tool that runs +-/* operations\ncalc_tool = load_tool(\"ybelkada/simple-calculator\")\n\n# python interpreter that executes program and returns outputs\npy_tool = load_tool(\"lvwerra/python-interpreter\")\n\n# wikipedia search index that returns best search match\nwiki_tool = load_tool(\"vwxyzjn/pyserini-wikipedia-kilt-doc\")\n```\n\nThese tools are either loaded from the hub or from a local folder. Using the tool is as simple as calling them with a text query:\n\n```Python\ncalc_tool(\"1/2\")\n>>> \"0.5\"\n```\n\nNote that both input and return values are strings to enable easy usage with a language model.\n\n### Custom Tools\n\nThe following is an example of a tool that adds two integers:\n\n```Python\ndef add(text):\n    int_1, int_2 = text.split(\"+\")\n    result = int(int_1) + int(int_2)\n    return str(result)\n\nprint(add(\"1+1\"))\n>>> \"2\"\n```\n\nWe looked at basic examples such as a calculator but the principle holds for more complex tools as well such as a web search tool where you input the query and get the search results in return. Now let's look at how the model can use the tools with the call syntax.\n\n### Call syntax\n\nIn order to have a unified way for the model to call a tool we created a simple syntax that looks as follows:\n\n```python\n\"<request><TOOL_NAME>QUERY<call>TOOL_RESPONSE<response>\"\n```\n\nThere are a few special tokens involved so let's decompose it: First the model can signal that it wants to use a tool by emitting the `<request>` token. After that we want to know the name of the tool to call which is done by enclosing the tool name with `<>` brackets. Once we know which tool to call the tool query follows which is in free text form. The `<call>` tokens signifies the end of the query and stops the model generation. At this point the model output is parsed and the query sent to the tool. The environment appends the tool response to the string followed by the `<response>` token to show the end the tool output."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d2fe3676-ee89-4ad4-a612-728e4afddaf8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/text_environments", "title": "Text Environments", "tokens": 2275, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d2fe3676-ee89-4ad4-a612-728e4afddaf8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1bd6dc94-02bd-4f50-b752-0cbe7044b926\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cba40e36c2453aef7be446350c9a4c783340bf7cb3c68911d89d32228a01679\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d1fb3764-3e95-458d-8619-e8ed5596061c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"aa99e54e9b58af91518e6b62d07f498e5d865bd45d279b6525a6c1689efe5a72\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2856b7ac-e619-475a-a684-b08c3bc8f358\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bb1f706289acdc264259e5c8683af676cf5c1f1d5ebe5e9557cabab79d140b16\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1787, \"end_char_idx\": 3750, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "ref_doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "chroma:document": "Using the tool is as simple as calling them with a text query:\n\n```Python\ncalc_tool(\"1/2\")\n>>> \"0.5\"\n```\n\nNote that both input and return values are strings to enable easy usage with a language model.\n\n### Custom Tools\n\nThe following is an example of a tool that adds two integers:\n\n```Python\ndef add(text):\n    int_1, int_2 = text.split(\"+\")\n    result = int(int_1) + int(int_2)\n    return str(result)\n\nprint(add(\"1+1\"))\n>>> \"2\"\n```\n\nWe looked at basic examples such as a calculator but the principle holds for more complex tools as well such as a web search tool where you input the query and get the search results in return. Now let's look at how the model can use the tools with the call syntax.\n\n### Call syntax\n\nIn order to have a unified way for the model to call a tool we created a simple syntax that looks as follows:\n\n```python\n\"<request><TOOL_NAME>QUERY<call>TOOL_RESPONSE<response>\"\n```\n\nThere are a few special tokens involved so let's decompose it: First the model can signal that it wants to use a tool by emitting the `<request>` token. After that we want to know the name of the tool to call which is done by enclosing the tool name with `<>` brackets. Once we know which tool to call the tool query follows which is in free text form. The `<call>` tokens signifies the end of the query and stops the model generation. At this point the model output is parsed and the query sent to the tool. The environment appends the tool response to the string followed by the `<response>` token to show the end the tool output.\n\nLet's look at the concrete example of the calculator and assume its name is `Calculator` (more on how the name of a tool is inferred later):\n\n```python\n\"<request><Calculator>1/2<call>0.5<response>\"\n```\n\nFinally, the episode is ended and generation stops when the model generates `<submit>` which marks the interaction as completed.\n\nNow let's have a look how we can create a new text environment!\n\n## Create a `TextEnvironment`"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2856b7ac-e619-475a-a684-b08c3bc8f358
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/text_environments", "title": "Text Environments", "tokens": 2275, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2856b7ac-e619-475a-a684-b08c3bc8f358\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1bd6dc94-02bd-4f50-b752-0cbe7044b926\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cba40e36c2453aef7be446350c9a4c783340bf7cb3c68911d89d32228a01679\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d2fe3676-ee89-4ad4-a612-728e4afddaf8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"68e7272dd4854a20ed5fe9561e08056f92e0709a78b56b2d560388e7dc10b8fd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0c060dcd-0fed-4dac-b806-a93a098942fa\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"17b2ccbf885a65d2c032202b9c56c891124e402961945a79746989eeb9834b78\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2416, \"end_char_idx\": 5679, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "ref_doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "chroma:document": "Now let's look at how the model can use the tools with the call syntax.\n\n### Call syntax\n\nIn order to have a unified way for the model to call a tool we created a simple syntax that looks as follows:\n\n```python\n\"<request><TOOL_NAME>QUERY<call>TOOL_RESPONSE<response>\"\n```\n\nThere are a few special tokens involved so let's decompose it: First the model can signal that it wants to use a tool by emitting the `<request>` token. After that we want to know the name of the tool to call which is done by enclosing the tool name with `<>` brackets. Once we know which tool to call the tool query follows which is in free text form. The `<call>` tokens signifies the end of the query and stops the model generation. At this point the model output is parsed and the query sent to the tool. The environment appends the tool response to the string followed by the `<response>` token to show the end the tool output.\n\nLet's look at the concrete example of the calculator and assume its name is `Calculator` (more on how the name of a tool is inferred later):\n\n```python\n\"<request><Calculator>1/2<call>0.5<response>\"\n```\n\nFinally, the episode is ended and generation stops when the model generates `<submit>` which marks the interaction as completed.\n\nNow let's have a look how we can create a new text environment!\n\n## Create a `TextEnvironment`\n\n\n```python\nprompt = \"\"\"\\\nWhat is 13-3?\n<request><SimpleCalculatorTool>13-3<call>10.0<response>\nResult=10<submit>\n\"\"\"\n\ndef reward_fn(result, answer):\n    \"\"\"Simplified reward function returning 1 if result matches answer and 0 otherwise.\"\"\"\n    result_parsed = result.split(\"=\")[1].split(\"<\")[0]\n    return int(result_parsed==answer)\n\ntext_env = TextEnvironemnt(\n    model=model, \n    tokenizer=tokenizer,\n    tools= {\"SimpleCalculatorTool\": load_tool(\"ybelkada/simple-calculator\")},\n    reward_fn=exact_match_reward,\n    prompt=prompt, \n    max_turns=1\n    max_tool_response=100\n    generation_kwargs={\"do_sample\": \"true\"}\n)\n```\n\nLet's decompose the settings:\n\n| Argument           | Description     |\n|:-------------------|:----------------|\n| `model`            | Language model to interact with the environment and generate requests. |\n| `tokenizer`        | Tokenizer of language model handling tokenization of strings. |\n| `tools`            | `list` of `dict` of tools. If former the name of the tool is inferred from class name and otherwise it's the keys of the dictionary.|\n| `reward_fn`        | A function that takes a string as input and returns. Can have extra arguments that are passed to `.run()` such as ground truth.|\n| `prompt`           | Prompt to prepend to every task. Usually a few examples to demonstrate to the model how to use the tools in a few-shot fashion. |\n| `max_turns`        | Maximum number of interactions between model and tools before episode ends.|\n| `max_tool_response`| The tool response is truncated to this number to avoid running out of model context.|\n| `max_length`       |  The maximum number of tokens to allow in an episode. |\n| `generation_kwargs`| Generation settings used by the language model. |\n\nYou can customize the environment to your needs and add custom tools and settings. Let's see how you can use the environment to have the model interact with the available tools!"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 0c060dcd-0fed-4dac-b806-a93a098942fa
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/text_environments", "title": "Text Environments", "tokens": 2275, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"0c060dcd-0fed-4dac-b806-a93a098942fa\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1bd6dc94-02bd-4f50-b752-0cbe7044b926\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cba40e36c2453aef7be446350c9a4c783340bf7cb3c68911d89d32228a01679\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2856b7ac-e619-475a-a684-b08c3bc8f358\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d04dff1b262b088aaced77bf4ebd09333d20b652df91598b0acc2a3a0228503b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1956e205-aa6e-47aa-be20-621c4288f6ab\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"28128b023640b5743d3b6eaec26b74aa0fe1279ea40e7133b9a2d07b085d2fb3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5682, \"end_char_idx\": 6957, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "ref_doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "chroma:document": "## Run an Episode\n\nTo run a set of queries through the text environment one can simply use the `run` method.\n\n```python\nqueries = [\"What is 1/2?\"]\nanswers = [\"0.5\"]\n\nqueries, responses, masks, rewards, histories = text_env.run(queries, answers=answers)\n```\n\nThis will execute the model/tool feedback loop for each query until either no tool is called anymore, the maximum number of turns is reached or to maximum number of tokens in an episode is exceeded. The extra `kwargs` (e.g. `answers=answers` above) passed to `run` will be passed on to the reward function.\n\nThere are five objects that are returned by `run`: \n\n- `queries`: a list of the tokenized queries\n- `responses`: all tokens that have been generated withing the environment including model and tool tokens\n- `masks`: mask that indicates which tokens have been generated by the model and which tokens are generated by the tool\n- `rewards`: a list of reward for each query/response\n- `histories`: list of `TextHistory` objects, which are useful objects containing all the above and also the text equivalents\n\nThe masks are crucial for training as we don't want to optimize tokens that the model has not generated which are tokens produced by the tools.\n\nNext, we'll train a PPO step with the generated responses!"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 1956e205-aa6e-47aa-be20-621c4288f6ab
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/text_environments", "title": "Text Environments", "tokens": 2275, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"1956e205-aa6e-47aa-be20-621c4288f6ab\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1bd6dc94-02bd-4f50-b752-0cbe7044b926\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9cba40e36c2453aef7be446350c9a4c783340bf7cb3c68911d89d32228a01679\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0c060dcd-0fed-4dac-b806-a93a098942fa\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/text_environments\", \"title\": \"Text Environments\", \"tokens\": 2275, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d5b1553d3e64b05e600e681e80eec462106c731810d796427466b1b208c44875\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5682, \"end_char_idx\": 10343, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "ref_doc_id": "1bd6dc94-02bd-4f50-b752-0cbe7044b926", "chroma:document": "## Run an Episode\n\nTo run a set of queries through the text environment one can simply use the `run` method.\n\n```python\nqueries = [\"What is 1/2?\"]\nanswers = [\"0.5\"]\n\nqueries, responses, masks, rewards, histories = text_env.run(queries, answers=answers)\n```\n\nThis will execute the model/tool feedback loop for each query until either no tool is called anymore, the maximum number of turns is reached or to maximum number of tokens in an episode is exceeded. The extra `kwargs` (e.g. `answers=answers` above) passed to `run` will be passed on to the reward function.\n\nThere are five objects that are returned by `run`: \n\n- `queries`: a list of the tokenized queries\n- `responses`: all tokens that have been generated withing the environment including model and tool tokens\n- `masks`: mask that indicates which tokens have been generated by the model and which tokens are generated by the tool\n- `rewards`: a list of reward for each query/response\n- `histories`: list of `TextHistory` objects, which are useful objects containing all the above and also the text equivalents\n\nThe masks are crucial for training as we don't want to optimize tokens that the model has not generated which are tokens produced by the tools.\n\nNext, we'll train a PPO step with the generated responses!\n\n\n### Train\nTraining on episodes from the `TextEnvironment` is straight forward and simply requires forwarding all the returned variables except the `TextHistory` objects to the `step` method:\n\n```python\ntrain_stats = ppo_trainer.step(queries, responses, rewards, masks)\n```\n\n## `TextHistory`\n\nThe `TextHistory` object stores the interactions between the model and the text environment. It stores tokens and text generated in each turn and their source in each turn (model or system) as well as rewards. Let's go through the class attributes and methods.\n\n### Attributes\n\nThe following table summarises the available attributes of the `TextEnvironment` class:\n\n| Attribute           | Description     |\n|:-------------------|:----------------|\n| `text`             | The full string of the text generated in the text environment with both model and system generated text. |\n| `text_spans`       | A list of tuples with the spans for each model or system generated text segment. |\n| `system_spans`     | A list of boolean values indicating if the segment is model or system generated. |\n| `tokens`           | All tokens generated in text environment with both model and system generated tokens. |\n| `token_spans`      | Similar to `text_spans` the `token_spans` indicate the boundaries of model andsystem generated tokens. |\n| `token_masks`      | The token masks can be used to ignore system generated tokens by masking them. |\n| `completed`        | Indicates if the interaction with the environment has completed. |\n| `truncated`        | Indicates if the interaction with the environment has completed because max length was reached. |\n\nWith these attributes you can reconstruct every interaction of the model with the `TextEnvironment`. The `TextHistory` also lets you visualize the text history. Let's have a look!\n\n### Visualization\n\nWhen the model interacts inside the `TextEnvironment` it can be useful to visualize and separate which parts of the text outputs were generated by the model and which parts come from the system and tools. For that purpose there are the two methods [`TextHistory.show_text`] and [`TextHistory.show_tokens`]. They print the text and tokens respectively and highlight the various segments using the [`rich` libray](https://github.com/Textualize/rich) (make sure to install it before using these methods).\n\nYou can see that the prompt is highlighted in gray, whereas system segments such as query and tool responses are highlighted in green. All segments generated by the model are highlighted in blue and in addition to the pure text output the reward is displayed as additional text in plum. Here an example of `show_text`:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/textenv_show_text.png\" width=600>\n</div>\n\nSometimes there can be tricky tokenization related issues that are hidden when showing the decoded text. Thus `TextHistory` also offers an option to display the same highlighting on the tokens directly with `show_tokens`:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/textenv_show_tokens.png\" width=800>\n</div>\n\nNote that you can turn on the colour legend by passing `show_legend=True`.\n\n## API Documentation\n\n[[autodoc]] TextEnvironment\n\n[[autodoc]] TextHistory"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 66bb9ddb-cc2d-49bf-9dc9-be0ef6cf478c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"66bb9ddb-cc2d-49bf-9dc9-be0ef6cf478c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8cc6d408-461b-4860-b73e-384d51cbedcd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3055cc8b574567cd6e227ac35bbc0f091e917f46598716fbdaf37b2dc5504e22\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2927, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "# DPO Trainer\n\nTRL supports the DPO Trainer for training language models from preference data, as described in the paper [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290) by Rafailov et al., 2023. For a full example have a look at  [`examples/scripts/dpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo.py).\n\nThe first step as always is to train your SFT model, to ensure the data we train on is in-distribution for the DPO algorithm.\n\n## How DPO works\n\nFine-tuning a language model via DPO consists of two steps and is easier than PPO:\n\n1. **Data collection**: Gather a preference dataset with positive and negative selected pairs of generation, given a prompt.\n2. **Optimization**: Maximize the log-likelihood of the DPO loss directly.\n\nDPO-compatible datasets can be found with [the tag `dpo` on Hugging Face Hub](https://huggingface.co/datasets?other=dpo). You can also explore the [librarian-bots/direct-preference-optimization-datasets](https://huggingface.co/collections/librarian-bots/direct-preference-optimization-datasets-66964b12835f46289b6ef2fc) Collection to identify datasets that are likely to support DPO training.\n\nThis process is illustrated in the sketch below (from [figure 1 of the original paper](https://huggingface.co/papers/2305.18290)):\n\n<img width=\"835\" alt=\"Screenshot 2024-03-19 at 12 39 41\" src=\"https://github.com/huggingface/trl/assets/49240599/9150fac6-3d88-4ca2-8ec6-2a6f3473216d\">\n\nRead more about DPO algorithm in the [original paper](https://huggingface.co/papers/2305.18290).\n\n\n## Expected dataset format\n\nThe DPO trainer expects a very specific format for the dataset. Since the model will be trained to directly optimize the preference of which sentence is the most relevant, given two sentences. We provide an example from the [`Anthropic/hh-rlhf`](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset below:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/rlhf-antropic-example.png\", width=\"50%\">\n</div>\n\nTherefore the final dataset object should contain these 3 entries if you use the default [`DPODataCollatorWithPadding`] data collator. The entries should be named:\n\n- `prompt`\n- `chosen`\n- `rejected`\n\nfor example:\n\n```py\ndpo_dataset_dict = {\n    \"prompt\": [\n        \"hello\",\n        \"how are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"chosen\": [\n        \"hi nice to meet you\",\n        \"I am fine\",\n        \"My name is Mary\",\n        \"My name is Mary\",\n        \"Python\",\n        \"Python\",\n        \"Java\",\n    ],\n    \"rejected\": [\n        \"leave me alone\",\n        \"I am not fine\",\n        \"Whats it to you?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 8cc6d408-461b-4860-b73e-384d51cbedcd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"8cc6d408-461b-4860-b73e-384d51cbedcd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"66bb9ddb-cc2d-49bf-9dc9-be0ef6cf478c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c80b0858683d9175d9f85f5fa0fe69dc92e1419c67c6388d982eb225ad8a9d12\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b30c1e2b-b3cc-465b-8b1f-b54217723fd2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6dbd6c623818be1800f305ff4ba17f6cde5977acd03d69a1d989d8323fa09cc7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1613, \"end_char_idx\": 4799, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "## Expected dataset format\n\nThe DPO trainer expects a very specific format for the dataset. Since the model will be trained to directly optimize the preference of which sentence is the most relevant, given two sentences. We provide an example from the [`Anthropic/hh-rlhf`](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset below:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/rlhf-antropic-example.png\", width=\"50%\">\n</div>\n\nTherefore the final dataset object should contain these 3 entries if you use the default [`DPODataCollatorWithPadding`] data collator. The entries should be named:\n\n- `prompt`\n- `chosen`\n- `rejected`\n\nfor example:\n\n```py\ndpo_dataset_dict = {\n    \"prompt\": [\n        \"hello\",\n        \"how are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"chosen\": [\n        \"hi nice to meet you\",\n        \"I am fine\",\n        \"My name is Mary\",\n        \"My name is Mary\",\n        \"Python\",\n        \"Python\",\n        \"Java\",\n    ],\n    \"rejected\": [\n        \"leave me alone\",\n        \"I am not fine\",\n        \"Whats it to you?\",\n        \"I dont have a name\",\n        \"Javascript\",\n        \"C++\",\n        \"C++\",\n    ],\n}\n```\n\nwhere the `prompt` contains the context inputs, `chosen` contains the corresponding chosen responses and `rejected` contains the corresponding negative (rejected) responses. As can be seen a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n[`DPOTrainer`] can be used to fine-tune visual language models (VLMs). In this case, the dataset must also contain the key `images`, and the trainer's `tokenizer` is the VLM's `processor`. For example, for Idefics2, the processor expects the dataset to have the following format:\n\nNote: Currently, VLM support is exclusive to Idefics2 and does not extend to other VLMs.\n\n```py\ndpo_dataset_dict = {\n    'images': [\n        [Image.open('beach.jpg')],\n        [Image.open('street.jpg')],\n    ],\n    'prompt': [\n        'The image <image> shows',\n        '<image> The image depicts',\n    ],\n    'chosen': [\n        'a sunny beach with palm trees.',\n        'a busy street with several cars and buildings.',\n    ],\n    'rejected': [\n        'a snowy mountain with skiers.',\n        'a calm countryside with green fields.',\n    ],\n}\n```\n\n## Expected model format\n\nThe DPO trainer expects a model of `AutoModelForCausalLM` or `AutoModelForVision2Seq`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `DPOTrainer`\n\nFor a detailed example have a look at the `examples/scripts/dpo.py` script. At a high level we need to initialize the [`DPOTrainer`] with a `model` we wish to train, a reference `ref_model` which we will use to calculate the implicit rewards of the preferred and rejected response, the `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b30c1e2b-b3cc-465b-8b1f-b54217723fd2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b30c1e2b-b3cc-465b-8b1f-b54217723fd2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8cc6d408-461b-4860-b73e-384d51cbedcd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"59b83027224fb6a9d4919f76a6f90d07aed850acab571dd47da6b48099b15bed\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a6cd65e6-ecf9-4ada-a2ff-92ea04f473bc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"67c8c02a82997deb2bae081d05981eef1a1101fd136ba240759a38b830bbdc2a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3200, \"end_char_idx\": 6271, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "As can be seen a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n[`DPOTrainer`] can be used to fine-tune visual language models (VLMs). In this case, the dataset must also contain the key `images`, and the trainer's `tokenizer` is the VLM's `processor`. For example, for Idefics2, the processor expects the dataset to have the following format:\n\nNote: Currently, VLM support is exclusive to Idefics2 and does not extend to other VLMs.\n\n```py\ndpo_dataset_dict = {\n    'images': [\n        [Image.open('beach.jpg')],\n        [Image.open('street.jpg')],\n    ],\n    'prompt': [\n        'The image <image> shows',\n        '<image> The image depicts',\n    ],\n    'chosen': [\n        'a sunny beach with palm trees.',\n        'a busy street with several cars and buildings.',\n    ],\n    'rejected': [\n        'a snowy mountain with skiers.',\n        'a calm countryside with green fields.',\n    ],\n}\n```\n\n## Expected model format\n\nThe DPO trainer expects a model of `AutoModelForCausalLM` or `AutoModelForVision2Seq`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `DPOTrainer`\n\nFor a detailed example have a look at the `examples/scripts/dpo.py` script. At a high level we need to initialize the [`DPOTrainer`] with a `model` we wish to train, a reference `ref_model` which we will use to calculate the implicit rewards of the preferred and rejected response, the `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above. Note that the `model` and `ref_model` need to have the same architecture (ie decoder only or encoder-decoder).\n\n```py\ntraining_args = DPOConfig(\n    beta=0.1,\n)\ndpo_trainer = DPOTrainer(\n    model,\n    ref_model,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,  # for visual language models, use tokenizer=processor instead\n)\n```\n\nAfter this one can then call:\n\n```py\ndpo_trainer.train()\n```\n\nNote that the `beta` is the temperature parameter for the DPO loss, typically something in the range of `0.1` to `0.5`. We ignore the reference model as `beta` -> 0.\n\n## Loss functions\n\nGiven the preference data, we can fit a binary classifier according to the Bradley-Terry model and in fact the [DPO](https://huggingface.co/papers/2305.18290) authors propose the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression. To use this loss, set the `loss_type=\"sigmoid\"` (default) in the [`DPOConfig`].\n\nThe [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. To use this loss, set the `loss_type=\"hinge\"` in the [`DPOConfig`]. In this case, the `beta` is the reciprocal of the margin.\n\nThe [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the DPO algorithms and identify an issue with overfitting and propose an alternative loss."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: a6cd65e6-ecf9-4ada-a2ff-92ea04f473bc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"a6cd65e6-ecf9-4ada-a2ff-92ea04f473bc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b30c1e2b-b3cc-465b-8b1f-b54217723fd2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"fbb68b773b357c06308b9ac19d421892f77f1eac8d8bdfba17c75faa6149afb8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9be6df82-e7f2-48ca-b8f9-ff3ef89132bd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"82690f79474a2cde7c54a0c4708715cbc7762a8aef3734a8e8ad1fc9d2dd0822\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4800, \"end_char_idx\": 7665, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "Note that the `model` and `ref_model` need to have the same architecture (ie decoder only or encoder-decoder).\n\n```py\ntraining_args = DPOConfig(\n    beta=0.1,\n)\ndpo_trainer = DPOTrainer(\n    model,\n    ref_model,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,  # for visual language models, use tokenizer=processor instead\n)\n```\n\nAfter this one can then call:\n\n```py\ndpo_trainer.train()\n```\n\nNote that the `beta` is the temperature parameter for the DPO loss, typically something in the range of `0.1` to `0.5`. We ignore the reference model as `beta` -> 0.\n\n## Loss functions\n\nGiven the preference data, we can fit a binary classifier according to the Bradley-Terry model and in fact the [DPO](https://huggingface.co/papers/2305.18290) authors propose the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression. To use this loss, set the `loss_type=\"sigmoid\"` (default) in the [`DPOConfig`].\n\nThe [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. To use this loss, set the `loss_type=\"hinge\"` in the [`DPOConfig`]. In this case, the `beta` is the reciprocal of the margin.\n\nThe [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the DPO algorithms and identify an issue with overfitting and propose an alternative loss. To use the loss set the `loss_type=\"ipo\"` in the [`DPOConfig`]. In this case, the `beta` is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair and thus the smaller the `beta` the larger this gaps is. As per the paper the loss is averaged over log-likelihoods of the completion (unlike DPO which is summed only). \n\nThe [cDPO](https://ericmitchell.ai/cdpo.pdf) is a tweak on the DPO loss where we assume that the preference labels are noisy with some probability. In this approach, the `label_smoothing` parameter in the [`DPOConfig`] is used to model the probability of existing label noise. To apply this conservative loss, set `label_smoothing` to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0).\n\nThe [EXO](https://huggingface.co/papers/2402.00856) authors propose to minimize the reverse KL instead of the negative log-sigmoid loss of DPO which corresponds to forward KL. To use the loss set the `loss_type=\"exo_pair\"` in the [`DPOConfig`]. Setting non-zero `label_smoothing` (default `1e-3`) leads to a simplified version of EXO on pair-wise preferences (see Eqn. (16) of the [EXO paper](https://huggingface.co/papers/2402.00856)). The full version of EXO uses `K>2` completions generated by the SFT policy, which becomes an unbiased estimator of the PPO objective (up to a constant) when `K` is sufficiently large."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 9be6df82-e7f2-48ca-b8f9-ff3ef89132bd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"9be6df82-e7f2-48ca-b8f9-ff3ef89132bd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a6cd65e6-ecf9-4ada-a2ff-92ea04f473bc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a29310d8d4624537b1ce53b409ea986fac6371c3c01bc6fd2d23122787c895c7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e0295bbb-5671-4100-963a-eb06ac47c473\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6165775d544b62425a9ae35d50b1c445960905a9a7baad8c1aca63e099cfae77\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6272, \"end_char_idx\": 9044, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "To use the loss set the `loss_type=\"ipo\"` in the [`DPOConfig`]. In this case, the `beta` is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair and thus the smaller the `beta` the larger this gaps is. As per the paper the loss is averaged over log-likelihoods of the completion (unlike DPO which is summed only). \n\nThe [cDPO](https://ericmitchell.ai/cdpo.pdf) is a tweak on the DPO loss where we assume that the preference labels are noisy with some probability. In this approach, the `label_smoothing` parameter in the [`DPOConfig`] is used to model the probability of existing label noise. To apply this conservative loss, set `label_smoothing` to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0).\n\nThe [EXO](https://huggingface.co/papers/2402.00856) authors propose to minimize the reverse KL instead of the negative log-sigmoid loss of DPO which corresponds to forward KL. To use the loss set the `loss_type=\"exo_pair\"` in the [`DPOConfig`]. Setting non-zero `label_smoothing` (default `1e-3`) leads to a simplified version of EXO on pair-wise preferences (see Eqn. (16) of the [EXO paper](https://huggingface.co/papers/2402.00856)). The full version of EXO uses `K>2` completions generated by the SFT policy, which becomes an unbiased estimator of the PPO objective (up to a constant) when `K` is sufficiently large.\n\nThe [NCA](https://huggingface.co/papers/2402.05369) authors shows that NCA optimizes the absolute likelihood for each response rather than the relative likelihood. To use the loss set the `loss_type=\"nca_pair\"` in the [`DPOConfig`].\n\nThe [Robust DPO](https://huggingface.co/papers/2403.00409) authors propose an unbiased estimate of the DPO loss that is robust to preference noise in the data. Like in cDPO, it assumes that the preference labels are noisy with some probability. In this approach, the `label_smoothing` parameter in the [`DPOConfig`] is used to model the probability of existing label noise. To apply this conservative loss, set `label_smoothing` to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0) and set the `loss_type=\"robust\"` in the [`DPOConfig`].\n\nThe [BCO](https://huggingface.co/papers/2404.04656) authors train a binary classifier whose logit serves as a reward so that the classifier maps {prompt, chosen completion} pairs to 1 and {prompt, rejected completion} pairs to 0. To use this loss, set the `loss_type=\"bco_pair\"` in the [`DPOConfig`].\n\nThe [TR-DPO](https://huggingface.co/papers/2404.09656) paper suggests syncing the reference model weights after every `ref_model_sync_steps` steps of SGD with weight `ref_model_mixup_alpha` during DPO training. To toggle this callback use the `sync_ref_model=True` in the [`DPOConfig`]."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: e0295bbb-5671-4100-963a-eb06ac47c473
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"e0295bbb-5671-4100-963a-eb06ac47c473\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9be6df82-e7f2-48ca-b8f9-ff3ef89132bd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0888d7329bfbdd907dea665cf3eda8cc394a25365f23bece11966af45d0bb894\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"16464480-c44b-4186-a33e-fba2d5bb9b29\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0e864a59c0f692cb20940f93f6722b77fe3a23624e9d34e083886061861de940\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7667, \"end_char_idx\": 10633, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "The [NCA](https://huggingface.co/papers/2402.05369) authors shows that NCA optimizes the absolute likelihood for each response rather than the relative likelihood. To use the loss set the `loss_type=\"nca_pair\"` in the [`DPOConfig`].\n\nThe [Robust DPO](https://huggingface.co/papers/2403.00409) authors propose an unbiased estimate of the DPO loss that is robust to preference noise in the data. Like in cDPO, it assumes that the preference labels are noisy with some probability. In this approach, the `label_smoothing` parameter in the [`DPOConfig`] is used to model the probability of existing label noise. To apply this conservative loss, set `label_smoothing` to a value greater than 0.0 (between 0.0 and 0.5; the default is 0.0) and set the `loss_type=\"robust\"` in the [`DPOConfig`].\n\nThe [BCO](https://huggingface.co/papers/2404.04656) authors train a binary classifier whose logit serves as a reward so that the classifier maps {prompt, chosen completion} pairs to 1 and {prompt, rejected completion} pairs to 0. To use this loss, set the `loss_type=\"bco_pair\"` in the [`DPOConfig`].\n\nThe [TR-DPO](https://huggingface.co/papers/2404.09656) paper suggests syncing the reference model weights after every `ref_model_sync_steps` steps of SGD with weight `ref_model_mixup_alpha` during DPO training. To toggle this callback use the `sync_ref_model=True` in the [`DPOConfig`].\n\nThe [RPO](https://huggingface.co/papers/2404.19733) paper implements an iterative preference tuning algorithm using a loss related to the RPO loss in this [paper](https://huggingface.co/papers/2405.16436) that essentially consists of a weighted SFT loss on the chosen preferences together with the DPO loss. To use this loss, set the `rpo_alpha` in the [`DPOConfig`] to an appropriate value. The paper suggests setting this weight to 1.0.\n\nThe [SPPO](https://huggingface.co/papers/2405.00675) authors claim that SPPO is capable of solving the Nash equilibrium iteratively by pushing the chosen rewards to be as large as 1/2 and the rejected rewards to be as small as -1/2 and can alleviate data sparsity issues. The implementation approximates this algorithm by employing hard label probabilities, assigning 1 to the winner and 0 to the loser. To use this loss, set the `loss_type=\"sppo_hard\"` in the [`DPOConfig`].\n\nThe [AOT](https://huggingface.co/papers/2406.05882) authors propose to use Distributional Preference Alignment Via Optimal Transport. Traditionally, the alignment algorithms use paired preferences at a sample level, which does not ensure alignment on the distributional level. AOT, on the other hand, can align LLMs on paired or unpaired preference data by making the reward distribution of the positive samples stochastically dominant in the first order on the distribution of negative samples. Specifically, `loss_type=\"aot\"` is appropriate for paired datasets, where each prompt has both chosen and rejected responses; `loss_type=\"aot_pair\"` is for unpaired datasets."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 16464480-c44b-4186-a33e-fba2d5bb9b29
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"16464480-c44b-4186-a33e-fba2d5bb9b29\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e0295bbb-5671-4100-963a-eb06ac47c473\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0b0dddae0e106738f1d9aea42f3a15fff62119859c9060cf67191e77a50211af\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2de2d4ce-d247-400d-bef6-2523d2f567b1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d2821d5317f04773f9937c9d4149af305e7bbd46700048b8d18b07eb8c4d4e89\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8969, \"end_char_idx\": 12339, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "To toggle this callback use the `sync_ref_model=True` in the [`DPOConfig`].\n\nThe [RPO](https://huggingface.co/papers/2404.19733) paper implements an iterative preference tuning algorithm using a loss related to the RPO loss in this [paper](https://huggingface.co/papers/2405.16436) that essentially consists of a weighted SFT loss on the chosen preferences together with the DPO loss. To use this loss, set the `rpo_alpha` in the [`DPOConfig`] to an appropriate value. The paper suggests setting this weight to 1.0.\n\nThe [SPPO](https://huggingface.co/papers/2405.00675) authors claim that SPPO is capable of solving the Nash equilibrium iteratively by pushing the chosen rewards to be as large as 1/2 and the rejected rewards to be as small as -1/2 and can alleviate data sparsity issues. The implementation approximates this algorithm by employing hard label probabilities, assigning 1 to the winner and 0 to the loser. To use this loss, set the `loss_type=\"sppo_hard\"` in the [`DPOConfig`].\n\nThe [AOT](https://huggingface.co/papers/2406.05882) authors propose to use Distributional Preference Alignment Via Optimal Transport. Traditionally, the alignment algorithms use paired preferences at a sample level, which does not ensure alignment on the distributional level. AOT, on the other hand, can align LLMs on paired or unpaired preference data by making the reward distribution of the positive samples stochastically dominant in the first order on the distribution of negative samples. Specifically, `loss_type=\"aot\"` is appropriate for paired datasets, where each prompt has both chosen and rejected responses; `loss_type=\"aot_pair\"` is for unpaired datasets. In a nutshell, `loss_type=\"aot\"` ensures that the log-likelihood ratio of chosen to rejected of the aligned model has higher quantiles than that ratio for the reference model. `loss_type=\"aot_pair\"` ensures that the chosen reward is higher on all quantiles than the rejected reward. Note that in both cases quantiles are obtained via sorting. To fully leverage the advantages of the AOT algorithm, it is important to maximize the per-GPU batch size.\n\nThe [APO](https://huggingface.co/papers/2408.06266) method introduces an \"anchored\" version of the alignment objective. There are two variants: `apo_zero` and `apo_down`. The `apo_zero` loss increases the likelihood of winning outputs while decreasing the likelihood of losing outputs, making it suitable when the model is less performant than the winning outputs. On the other hand, `apo_down` decreases the likelihood of both winning and losing outputs, but with a stronger emphasis on reducing the likelihood of losing outputs. This variant is more effective when the model is better than the winning outputs. To use these losses, set `loss_type=\"apo_zero\"` or `loss_type=\"apo_down\"` in the [`DPOConfig`].\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.\n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2de2d4ce-d247-400d-bef6-2523d2f567b1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2de2d4ce-d247-400d-bef6-2523d2f567b1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"16464480-c44b-4186-a33e-fba2d5bb9b29\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a8584920f22208fb2ae62dfac90bf06f616d08faca29732aa0cdbe0c130cfd55\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0b1d4f77-1c8d-4136-8612-2b2a162b79b1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2f2f3401303438286677321b849903805230026eabc072a3af5c8eae3b33010e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10634, \"end_char_idx\": 13802, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "In a nutshell, `loss_type=\"aot\"` ensures that the log-likelihood ratio of chosen to rejected of the aligned model has higher quantiles than that ratio for the reference model. `loss_type=\"aot_pair\"` ensures that the chosen reward is higher on all quantiles than the rejected reward. Note that in both cases quantiles are obtained via sorting. To fully leverage the advantages of the AOT algorithm, it is important to maximize the per-GPU batch size.\n\nThe [APO](https://huggingface.co/papers/2408.06266) method introduces an \"anchored\" version of the alignment objective. There are two variants: `apo_zero` and `apo_down`. The `apo_zero` loss increases the likelihood of winning outputs while decreasing the likelihood of losing outputs, making it suitable when the model is less performant than the winning outputs. On the other hand, `apo_down` decreases the likelihood of both winning and losing outputs, but with a stronger emphasis on reducing the likelihood of losing outputs. This variant is more effective when the model is better than the winning outputs. To use these losses, set `loss_type=\"apo_zero\"` or `loss_type=\"apo_down\"` in the [`DPOConfig`].\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.\n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001).\n\n## Logging\n\nWhile training and evaluating we record the following reward metrics:\n\n- `rewards/chosen`: the mean difference between the log probabilities of the policy model and the reference model for the chosen responses scaled by beta\n- `rewards/rejected`: the mean difference between the log probabilities of the policy model and the reference model for the rejected responses scaled by beta\n- `rewards/accuracies`: mean of how often the chosen rewards are > than the corresponding rejected rewards\n- `rewards/margins`: the mean difference between the chosen and corresponding rejected rewards\n\n## Accelerate DPO fine-tuning using `unsloth`\n\nYou can further accelerate QLoRA / LoRA (2x faster, 60% less memory) using the [`unsloth`](https://github.com/unslothai/unsloth) library that is fully compatible with `SFTTrainer`. Currently `unsloth` supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures. Some benchmarks for DPO listed below:\n\n| GPU      | Model     | Dataset    | \ud83e\udd17  | \ud83e\udd17 + Flash Attention 2 | \ud83e\udda5 Unsloth | \ud83e\udda5 VRAM saved |\n| -------- | --------- | ---------- | --- | ---------------------- | ---------- | ------------- |\n| A100 40G | Zephyr 7b | Ultra Chat | 1x  | 1.24x                  | **1.88x**  | -11.6%        |\n| Tesla T4 | Zephyr 7b | Ultra Chat | 1x  | 1.09x                  | **1.55x**  | -18.6%        |\n\nFirst install `unsloth` according to the [official documentation](https://github.com/unslothai/unsloth)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 0b1d4f77-1c8d-4136-8612-2b2a162b79b1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"0b1d4f77-1c8d-4136-8612-2b2a162b79b1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2de2d4ce-d247-400d-bef6-2523d2f567b1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"46a187979e3e05eacf2266ab6080e919567d486e14f9a4652c55e550bcd67482\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2e66ffba-672a-4521-ae34-c5cc29e5e490\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2add5cf67e740489fa805be7428ece7290dd9a68b998f6f6fc21287fe82fcfd7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12341, \"end_char_idx\": 14494, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "## Logging\n\nWhile training and evaluating we record the following reward metrics:\n\n- `rewards/chosen`: the mean difference between the log probabilities of the policy model and the reference model for the chosen responses scaled by beta\n- `rewards/rejected`: the mean difference between the log probabilities of the policy model and the reference model for the rejected responses scaled by beta\n- `rewards/accuracies`: mean of how often the chosen rewards are > than the corresponding rejected rewards\n- `rewards/margins`: the mean difference between the chosen and corresponding rejected rewards\n\n## Accelerate DPO fine-tuning using `unsloth`\n\nYou can further accelerate QLoRA / LoRA (2x faster, 60% less memory) using the [`unsloth`](https://github.com/unslothai/unsloth) library that is fully compatible with `SFTTrainer`. Currently `unsloth` supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures. Some benchmarks for DPO listed below:\n\n| GPU      | Model     | Dataset    | \ud83e\udd17  | \ud83e\udd17 + Flash Attention 2 | \ud83e\udda5 Unsloth | \ud83e\udda5 VRAM saved |\n| -------- | --------- | ---------- | --- | ---------------------- | ---------- | ------------- |\n| A100 40G | Zephyr 7b | Ultra Chat | 1x  | 1.24x                  | **1.88x**  | -11.6%        |\n| Tesla T4 | Zephyr 7b | Ultra Chat | 1x  | 1.09x                  | **1.55x**  | -18.6%        |\n\nFirst install `unsloth` according to the [official documentation](https://github.com/unslothai/unsloth). Once installed, you can incorporate unsloth into your workflow in a very simple manner; instead of loading `AutoModelForCausalLM`, you just need to load a `FastLanguageModel` as follows:\n\n```python\nimport torch\nfrom trl import DPOConfig, DPOTrainer\nfrom unsloth import FastLanguageModel\n\nmax_seq_length = 2048 # Supports automatic RoPE Scaling, so choose any number.\n\n# Load model\nmodel, tokenizer = FastLanguageModel.from_pretrained(\n    model_name = \"unsloth/zephyr-sft\",\n    max_seq_length = max_seq_length,\n    dtype = None, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n    load_in_4bit = True, # Use 4bit quantization to reduce memory usage. Can be False."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2e66ffba-672a-4521-ae34-c5cc29e5e490
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2e66ffba-672a-4521-ae34-c5cc29e5e490\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0b1d4f77-1c8d-4136-8612-2b2a162b79b1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"d8038986dc82e6b321714c7783329bcc82c5fd1d2924e7669de4f866fad33a01\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fda47293-062a-4fff-be81-47b3ed3fa145\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a54522f0a723bb9802e1ae88e5f8fb638db2e79c6ef0de645c0681adc89407ee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13167, \"end_char_idx\": 16032, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "Currently `unsloth` supports only Llama (Yi, TinyLlama, Qwen, Deepseek etc) and Mistral architectures. Some benchmarks for DPO listed below:\n\n| GPU      | Model     | Dataset    | \ud83e\udd17  | \ud83e\udd17 + Flash Attention 2 | \ud83e\udda5 Unsloth | \ud83e\udda5 VRAM saved |\n| -------- | --------- | ---------- | --- | ---------------------- | ---------- | ------------- |\n| A100 40G | Zephyr 7b | Ultra Chat | 1x  | 1.24x                  | **1.88x**  | -11.6%        |\n| Tesla T4 | Zephyr 7b | Ultra Chat | 1x  | 1.09x                  | **1.55x**  | -18.6%        |\n\nFirst install `unsloth` according to the [official documentation](https://github.com/unslothai/unsloth). Once installed, you can incorporate unsloth into your workflow in a very simple manner; instead of loading `AutoModelForCausalLM`, you just need to load a `FastLanguageModel` as follows:\n\n```python\nimport torch\nfrom trl import DPOConfig, DPOTrainer\nfrom unsloth import FastLanguageModel\n\nmax_seq_length = 2048 # Supports automatic RoPE Scaling, so choose any number.\n\n# Load model\nmodel, tokenizer = FastLanguageModel.from_pretrained(\n    model_name = \"unsloth/zephyr-sft\",\n    max_seq_length = max_seq_length,\n    dtype = None, # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n    load_in_4bit = True, # Use 4bit quantization to reduce memory usage. Can be False.\n    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n)\n\n# Do model patching and add fast LoRA weights\nmodel = FastLanguageModel.get_peft_model(\n    model,\n    r = 16,\n    target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n                      \"gate_proj\", \"up_proj\", \"down_proj\",],\n    lora_alpha = 16,\n    lora_dropout = 0, # Dropout = 0 is currently optimized\n    bias = \"none\",    # Bias = \"none\" is currently optimized\n    use_gradient_checkpointing = True,\n    random_state = 3407,\n)\n\ntraining_args = DPOConfig(\n    output_dir=\"./output\",\n    beta=0.1,\n)\n\ndpo_trainer = DPOTrainer(\n    model,\n    ref_model=None,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\ndpo_trainer.train()\n```\n\nThe saved model is fully compatible with Hugging Face's transformers library. Learn more about unsloth in their [official repository](https://github.com/unslothai/unsloth).\n\n## Reference model considerations with PEFT\n\nYou have three main options (plus several variants) for how the reference model works when using PEFT, assuming the model that you would like to further enhance with DPO was tuned using (Q)LoRA.\n\n1. Simply create two instances of the model, each loading your adapter - works fine but is very inefficient.\n2. Merge the adapter into the base model, create another adapter on top, then leave the `ref_model` param null, in which case DPOTrainer will unload the adapter for reference inference - efficient, but has potential downsides discussed below.\n3."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: fda47293-062a-4fff-be81-47b3ed3fa145
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"fda47293-062a-4fff-be81-47b3ed3fa145\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2e66ffba-672a-4521-ae34-c5cc29e5e490\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"7cd64588ccdacf377472a510bac96ff14fb87f3d42061aae8a3a53930f15e337\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"80a7228e-6438-4b6d-8a4f-4152ae941757\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9b1cd7d75a536a5d920eddefbbc8342059491d138841fb025a1a943dadca574b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14481, \"end_char_idx\": 17150, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "Can be False.\n    # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n)\n\n# Do model patching and add fast LoRA weights\nmodel = FastLanguageModel.get_peft_model(\n    model,\n    r = 16,\n    target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n                      \"gate_proj\", \"up_proj\", \"down_proj\",],\n    lora_alpha = 16,\n    lora_dropout = 0, # Dropout = 0 is currently optimized\n    bias = \"none\",    # Bias = \"none\" is currently optimized\n    use_gradient_checkpointing = True,\n    random_state = 3407,\n)\n\ntraining_args = DPOConfig(\n    output_dir=\"./output\",\n    beta=0.1,\n)\n\ndpo_trainer = DPOTrainer(\n    model,\n    ref_model=None,\n    args=training_args,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\ndpo_trainer.train()\n```\n\nThe saved model is fully compatible with Hugging Face's transformers library. Learn more about unsloth in their [official repository](https://github.com/unslothai/unsloth).\n\n## Reference model considerations with PEFT\n\nYou have three main options (plus several variants) for how the reference model works when using PEFT, assuming the model that you would like to further enhance with DPO was tuned using (Q)LoRA.\n\n1. Simply create two instances of the model, each loading your adapter - works fine but is very inefficient.\n2. Merge the adapter into the base model, create another adapter on top, then leave the `ref_model` param null, in which case DPOTrainer will unload the adapter for reference inference - efficient, but has potential downsides discussed below.\n3. Load the adapter twice with different names, then use `set_adapter` during training to swap between the adapter being DPO'd and the reference adapter - slightly less efficient compared to 2 (~adapter size VRAM overhead), but avoids the pitfalls.\n\n### Downsides to merging QLoRA before DPO (approach 2)\n\nAs suggested by [Benjamin Marie](https://medium.com/@bnjmn_marie/dont-merge-your-lora-adapter-into-a-4-bit-llm-65b6da287997), the best option for merging QLoRA adapters is to first dequantize the base model, then merge the adapter. Something similar to [this script](https://github.com/jondurbin/qlora/blob/main/qmerge.py).\n\nHowever, after using this approach, you will have an unquantized base model. Therefore, to use QLoRA for DPO, you will need to re-quantize the merged model or use the unquantized merge (resulting in higher memory demand).\n\n### Using option 3 - load the adapter twice\n\nTo avoid the downsides with option 2, you can load your fine-tuned adapter into the model twice, with different names, and set the model/ref adapter names in [`DPOTrainer`].\n\nFor example:\n\n```python\n# Load the base model."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 80a7228e-6438-4b6d-8a4f-4152ae941757
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/dpo_trainer", "title": "DPO Trainer", "tokens": 4570, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"80a7228e-6438-4b6d-8a4f-4152ae941757\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2d44c2ff-d5ef-4567-ad9e-bb309738b311\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c373a01b66f6d8dd0042e7ce819cfc28b3b25495657841605926550d68bd7330\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fda47293-062a-4fff-be81-47b3ed3fa145\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/dpo_trainer\", \"title\": \"DPO Trainer\", \"tokens\": 4570, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3cd8f5dd286ffb9f5ce25dce053fba94d925a283fda289f5a738faf9f88591e5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15678, \"end_char_idx\": 18248, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "ref_doc_id": "2d44c2ff-d5ef-4567-ad9e-bb309738b311", "chroma:document": "1. Simply create two instances of the model, each loading your adapter - works fine but is very inefficient.\n2. Merge the adapter into the base model, create another adapter on top, then leave the `ref_model` param null, in which case DPOTrainer will unload the adapter for reference inference - efficient, but has potential downsides discussed below.\n3. Load the adapter twice with different names, then use `set_adapter` during training to swap between the adapter being DPO'd and the reference adapter - slightly less efficient compared to 2 (~adapter size VRAM overhead), but avoids the pitfalls.\n\n### Downsides to merging QLoRA before DPO (approach 2)\n\nAs suggested by [Benjamin Marie](https://medium.com/@bnjmn_marie/dont-merge-your-lora-adapter-into-a-4-bit-llm-65b6da287997), the best option for merging QLoRA adapters is to first dequantize the base model, then merge the adapter. Something similar to [this script](https://github.com/jondurbin/qlora/blob/main/qmerge.py).\n\nHowever, after using this approach, you will have an unquantized base model. Therefore, to use QLoRA for DPO, you will need to re-quantize the merged model or use the unquantized merge (resulting in higher memory demand).\n\n### Using option 3 - load the adapter twice\n\nTo avoid the downsides with option 2, you can load your fine-tuned adapter into the model twice, with different names, and set the model/ref adapter names in [`DPOTrainer`].\n\nFor example:\n\n```python\n# Load the base model.\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    llm_int8_threshold=6.0,\n    llm_int8_has_fp16_weight=False,\n    bnb_4bit_compute_dtype=torch.bfloat16,\n    bnb_4bit_use_double_quant=True,\n    bnb_4bit_quant_type=\"nf4\",\n)\nmodel = AutoModelForCausalLM.from_pretrained(\n    \"mistralai/mixtral-8x7b-v0.1\",\n    load_in_4bit=True,\n    quantization_config=bnb_config,\n    attn_implementation=\"flash_attention_2\",\n    torch_dtype=torch.bfloat16,\n    device_map=\"auto\",\n)\nmodel.config.use_cache = False\n\n# Load the adapter.\nmodel = PeftModel.from_pretrained(\n    model,\n    \"/path/to/peft\",\n    is_trainable=True,\n    adapter_name=\"train\",\n)\n# Load the adapter a second time, with a different name, which will be our reference model.\nmodel.load_adapter(\"/path/to/peft\", adapter_name=\"reference\")\n\n# Initialize the trainer, without a ref_model param.\ntraining_args = DPOConfig(\n    model_adapter_name=\"train\",\n    ref_adapter_name=\"reference\",\n)\ndpo_trainer = DPOTrainer(\n    model,\n    args=training_args,\n    ...\n)\n```\n\n## DPOTrainer\n\n[[autodoc]] DPOTrainer\n\n## DPOConfig\n\n[[autodoc]] DPOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b7cf1519-3201-4669-bb10-ebc0dcab40fe
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/how_to_train", "title": "Training FAQ", "tokens": 1449, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b7cf1519-3201-4669-bb10-ebc0dcab40fe\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"74f24977-4b7c-40b9-91e5-d7cf1f262af2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"6ebac2186358630b480c3669534171db736f8a85e7bf06265b73412a24d6d236\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0f531903-5279-40ef-ba18-1ec86b774f91\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2ff3ed763d261bc0cd70d608b35e87663ad33bcc6e9fe5a18cb094b56372c19a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3653, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "ref_doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "chroma:document": "# Training FAQ\n\n## What Metrics Should I Look at?\n\nWhen performing classical supervised fine-tuning of language models, the loss (especially the validation loss) serves as a good indicator of the training progress. However, in Reinforcement Learning (RL), the loss becomes less informative about the model's performance, and its value may fluctuate while the actual performance improves.\n\nTo address this, we recommend focusing on two key metrics first:\n\n**Mean Reward**: The primary goal is to maximize the reward achieved by the model during RL training.\n**Objective KL Divergence**: KL divergence (Kullback-Leibler divergence) measures the dissimilarity between two probability distributions. In the context of RL training, we use it to quantify the difference between the current model and a reference model. Ideally, we want to keep the KL divergence between 0 and 10 to ensure the model's generated text remains close to what the reference model produces.\n\nHowever, there are more metrics that can be useful for debugging, checkout the [logging section](logging).\n\n## Why Do We Use a Reference Model, and What's the Purpose of KL Divergence?\n\nWhen training RL models, optimizing solely for reward may lead to unexpected behaviors, where the model exploits the environment in ways that don't align with good language generation. In the case of RLHF, we use a reward model trained to predict whether a generated text is highly ranked by humans.\n\nHowever, the RL model being optimized against the reward model may learn patterns that yield high reward but do not represent good language. This can result in extreme cases where the model generates texts with excessive exclamation marks or emojis to maximize the reward. In some worst-case scenarios, the model may generate patterns completely unrelated to natural language yet receive high rewards, similar to adversarial attacks.\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/kl-example.png\">\n<p style=\"text-align: center;\"> <b>Figure:</b> Samples without a KL penalty from <a href=\"https://huggingface.co/papers/1909.08593\">https://huggingface.co/papers/1909.08593</a>. </p>\n</div>\n\nTo address this issue, we add a penalty to the reward function based on the KL divergence between the current model and the reference model. By doing this, we encourage the model to stay close to what the reference model generates.\n\n## What Is the Concern with Negative KL Divergence?\n\nIf you generate text by purely sampling from the model distribution things work fine in general. But when you use the `generate` method there are a few caveats because it does not always purely sample depending on the settings which can cause KL-divergence to go negative. Essentially when the active model achieves `log_p_token_active < log_p_token_ref` we get negative KL-div. This can happen in a several cases:\n\n- **top-k sampling**: the model can smooth out the probability distribution causing the top-k tokens having a smaller probability than those of the reference model but they still are selected\n- **min_length**: this ignores the EOS token until `min_length` is reached. thus the model can assign a very low log prob to the EOS token and very high probs to all others until min_length is reached\n\nThese are just a few examples. Why is negative KL an issue? The total reward `R` is computed `R = r - beta * KL` so if the model can learn how to drive KL-divergence negative it effectively gets a positive reward. In many cases it can be much easier to exploit such a bug in the generation than actually learning the reward function."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 0f531903-5279-40ef-ba18-1ec86b774f91
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/how_to_train", "title": "Training FAQ", "tokens": 1449, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"0f531903-5279-40ef-ba18-1ec86b774f91\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"74f24977-4b7c-40b9-91e5-d7cf1f262af2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"6ebac2186358630b480c3669534171db736f8a85e7bf06265b73412a24d6d236\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b7cf1519-3201-4669-bb10-ebc0dcab40fe\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"2f3dcf2a430912397784191bb5e2cb8dadcbbd0d00f5730b7fc235688aa1c0a7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b3caf674-aa69-4efb-9142-72f9e3ff2ed7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3188cb470e03133d100f7841758739587a3d225e8ec70221ae6beec4ce40d0bf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2211, \"end_char_idx\": 5746, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "ref_doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "chroma:document": "</p>\n</div>\n\nTo address this issue, we add a penalty to the reward function based on the KL divergence between the current model and the reference model. By doing this, we encourage the model to stay close to what the reference model generates.\n\n## What Is the Concern with Negative KL Divergence?\n\nIf you generate text by purely sampling from the model distribution things work fine in general. But when you use the `generate` method there are a few caveats because it does not always purely sample depending on the settings which can cause KL-divergence to go negative. Essentially when the active model achieves `log_p_token_active < log_p_token_ref` we get negative KL-div. This can happen in a several cases:\n\n- **top-k sampling**: the model can smooth out the probability distribution causing the top-k tokens having a smaller probability than those of the reference model but they still are selected\n- **min_length**: this ignores the EOS token until `min_length` is reached. thus the model can assign a very low log prob to the EOS token and very high probs to all others until min_length is reached\n\nThese are just a few examples. Why is negative KL an issue? The total reward `R` is computed `R = r - beta * KL` so if the model can learn how to drive KL-divergence negative it effectively gets a positive reward. In many cases it can be much easier to exploit such a bug in the generation than actually learning the reward function. In addition the KL can become arbitrarily small thus the actual reward can be very small compared to it.\n\nSo how should you generate text for PPO training? Let's have a look!\n\n## How to generate text for training?\n\nIn order to avoid the KL issues described above we recommend to use the following settings:\n\n```python\ngeneration_kwargs = {\n    \"min_length\": -1, # don't ignore the EOS token (see above)\n    \"top_k\": 0.0, # no top-k sampling\n    \"top_p\": 1.0, # no nucleus sampling\n    \"do_sample\": True, # yes, we want to sample\n    \"pad_token_id\": tokenizer.eos_token_id, # most decoder models don't have a padding token - use EOS token instead\n    \"max_new_tokens\": 32, # specify how many tokens you want to generate at most\n}\n```\n\nWith these settings we usually don't encounter any issues. You can also experiments with other settings but if you encounter issues with negative KL-divergence try to go back to these and see if they persist.\n\n## How can debug your own use-case?\n\nDebugging the RL pipeline can be challenging due to its complexity. Here are some tips and suggestions to make the process easier:\n\n- **Start from a working example**: Begin with a working example from the trl repository and gradually modify it to fit your specific use-case. Changing everything at once can make it difficult to identify the source of potential issues. For example, you can start by replacing the model in the example and once you figure out the best hyperparameters try to switch to your dataset and reward model. If you change everything at once you won't know where a potential problem comes from.\n- **Start small, scale later**: Training large models can be very slow and take several hours or days until you see any improvement. For debugging this is not a convenient timescale so try to use small model variants during the development phase and scale up once that works. That being said you sometimes have to be careful as small models might not have the capacity to solve a complicated task either.\n- **Start simple**: Try to start with a minimal example and build complexity from there."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b3caf674-aa69-4efb-9142-72f9e3ff2ed7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/how_to_train", "title": "Training FAQ", "tokens": 1449, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b3caf674-aa69-4efb-9142-72f9e3ff2ed7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"74f24977-4b7c-40b9-91e5-d7cf1f262af2\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"6ebac2186358630b480c3669534171db736f8a85e7bf06265b73412a24d6d236\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0f531903-5279-40ef-ba18-1ec86b774f91\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/how_to_train\", \"title\": \"Training FAQ\", \"tokens\": 1449, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9f249c02283505b22b134f7d0581ccc88d22f8722150f29f97ce6e00971837aa\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4447, \"end_char_idx\": 6922, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "ref_doc_id": "74f24977-4b7c-40b9-91e5-d7cf1f262af2", "chroma:document": "You can also experiments with other settings but if you encounter issues with negative KL-divergence try to go back to these and see if they persist.\n\n## How can debug your own use-case?\n\nDebugging the RL pipeline can be challenging due to its complexity. Here are some tips and suggestions to make the process easier:\n\n- **Start from a working example**: Begin with a working example from the trl repository and gradually modify it to fit your specific use-case. Changing everything at once can make it difficult to identify the source of potential issues. For example, you can start by replacing the model in the example and once you figure out the best hyperparameters try to switch to your dataset and reward model. If you change everything at once you won't know where a potential problem comes from.\n- **Start small, scale later**: Training large models can be very slow and take several hours or days until you see any improvement. For debugging this is not a convenient timescale so try to use small model variants during the development phase and scale up once that works. That being said you sometimes have to be careful as small models might not have the capacity to solve a complicated task either.\n- **Start simple**: Try to start with a minimal example and build complexity from there. Your use-case might require for example a complicated reward function consisting of many different rewards - try to use one signal first and see if you can optimize that and then add more complexity after that.\n- **Inspect the generations**: It's always a good idea to inspect what the model is generating. Maybe there is a bug in your post-processing or your prompt. Due to bad settings you might cut-off generations too soon. These things are very hard to see on the metrics but very obvious if you look at the generations.\n- **Inspect the reward model**: If you reward is not improving over time maybe there's an issue with the reward model. You can look at extreme cases to see if it does what it should: e.g. in the sentiment case you can check if simple positive and negative examples really get different rewards. And you can look at the distribution of your dataset. Finally, maybe the reward is dominated by the query which the model can't affect so you might need to normalize this (e.g. reward of query+response minus reward of the query).\n\nThese are just a few tips that we find helpful - if you have more useful tricks feel free to open a PR to add them as well!"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 3843dcd8-b4d8-4e66-ad08-e458c86cf6a2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ddpo_trainer", "title": "Denoising Diffusion Policy Optimization", "tokens": 1924, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"3843dcd8-b4d8-4e66-ad08-e458c86cf6a2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"99e5bce9-55ed-41e1-8560-cc983baf3e9f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b0634b27587b9aebba82d04ac07e3dc86d3aadb8d744491d5e0c1e44640d14c3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1fcb2598-9dbd-4487-b72e-00c275088ae3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"65e56be18eb8c8728163c2ae6f01d6205442084a879aea958e2e7b543d6de1d1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3533, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "ref_doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "chroma:document": "# Denoising Diffusion Policy Optimization\n## The why\n\n| Before | After DDPO finetuning |\n| --- | --- |\n| <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/pre_squirrel.png\"/></div> |  <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/post_squirrel.png\"/></div> |\n| <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/pre_crab.png\"/></div> |  <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/post_crab.png\"/></div> |\n|  <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/pre_starfish.png\"/></div> |  <div style=\"text-align: center\"><img src=\"https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/post_starfish.png\"/></div> |\n\n\n## Getting started with Stable Diffusion finetuning with reinforcement learning\n\nThe machinery for finetuning of Stable Diffusion models with reinforcement learning makes heavy use of HuggingFace's `diffusers`\nlibrary. A reason for  stating this is that getting started requires a bit of familiarity with the `diffusers` library concepts, mainly two of them - pipelines and schedulers.\nRight out of the box (`diffusers` library), there isn't a `Pipeline` nor a `Scheduler` instance that is suitable for finetuning with reinforcement learning. Some adjustments need to made. \n\nThere is a pipeline interface that is provided by this library that is required to be implemented to be used with the `DDPOTrainer`, which is the main machinery for fine-tuning Stable Diffusion with reinforcement learning. **Note: Only the StableDiffusion architecture is supported at this point.**\nThere is a default implementation of this interface that you can use out of the box. Assuming the default implementation is sufficient and/or to get things moving, refer to the training example alongside this guide. \n\nThe point of the interface is to fuse the pipeline and the scheduler into one object which allows for minimalness in terms of having the constraints all in one place. The interface was designed in hopes of catering to pipelines and schedulers beyond the examples in this repository and elsewhere at this time of writing. Also the scheduler step is a method of this pipeline interface and this may seem redundant given that the raw scheduler is accessible via the interface but this is the only way to constrain the scheduler step output to an output type befitting of the algorithm at hand (DDPO).\n\nFor a more detailed look into the interface and the associated default implementation, go [here](https://github.com/lvwerra/trl/tree/main/trl/models/modeling_sd_base.py)\n\nNote that the default implementation has a LoRA implementation path and a non-LoRA based implementation path. The LoRA flag enabled by default and this can be turned off by passing in the flag to do so. LORA based training is faster and the LORA associated model hyperparameters responsible for model convergence aren't as finicky as non-LORA based training.\n\nAlso in addition, there is the expectation of providing a reward function and a prompt function. The reward function is used to evaluate the generated images  and the prompt function is used to generate the prompts that are used to generate the images."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 1fcb2598-9dbd-4487-b72e-00c275088ae3
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ddpo_trainer", "title": "Denoising Diffusion Policy Optimization", "tokens": 1924, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"1fcb2598-9dbd-4487-b72e-00c275088ae3\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"99e5bce9-55ed-41e1-8560-cc983baf3e9f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b0634b27587b9aebba82d04ac07e3dc86d3aadb8d744491d5e0c1e44640d14c3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3843dcd8-b4d8-4e66-ad08-e458c86cf6a2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c681e12a96ae01291fe53f9ee419d10f514490074d00ed510ad4d5d4de21a2eb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"aaeeb883-dc2d-4bda-a1c7-9de085a83a20\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7a0efa8b8570098adcfd37838526041a85b483fd033cfb26913426ea19d94b6f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1601, \"end_char_idx\": 4387, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "ref_doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "chroma:document": "Some adjustments need to made. \n\nThere is a pipeline interface that is provided by this library that is required to be implemented to be used with the `DDPOTrainer`, which is the main machinery for fine-tuning Stable Diffusion with reinforcement learning. **Note: Only the StableDiffusion architecture is supported at this point.**\nThere is a default implementation of this interface that you can use out of the box. Assuming the default implementation is sufficient and/or to get things moving, refer to the training example alongside this guide. \n\nThe point of the interface is to fuse the pipeline and the scheduler into one object which allows for minimalness in terms of having the constraints all in one place. The interface was designed in hopes of catering to pipelines and schedulers beyond the examples in this repository and elsewhere at this time of writing. Also the scheduler step is a method of this pipeline interface and this may seem redundant given that the raw scheduler is accessible via the interface but this is the only way to constrain the scheduler step output to an output type befitting of the algorithm at hand (DDPO).\n\nFor a more detailed look into the interface and the associated default implementation, go [here](https://github.com/lvwerra/trl/tree/main/trl/models/modeling_sd_base.py)\n\nNote that the default implementation has a LoRA implementation path and a non-LoRA based implementation path. The LoRA flag enabled by default and this can be turned off by passing in the flag to do so. LORA based training is faster and the LORA associated model hyperparameters responsible for model convergence aren't as finicky as non-LORA based training.\n\nAlso in addition, there is the expectation of providing a reward function and a prompt function. The reward function is used to evaluate the generated images  and the prompt function is used to generate the prompts that are used to generate the images.\n\n## Getting started with `examples/scripts/ddpo.py`\n\nThe `ddpo.py` script is a working example of using the `DDPO` trainer to finetune a Stable Diffusion model. This example explicitly configures a small subset of the overall parameters associated with the config object (`DDPOConfig`).\n\n**Note:** one A100 GPU is recommended to get this running. Anything below a A100 will not be able to run this example script and even if it does via relatively smaller sized parameters, the results will most likely be poor.\n\nAlmost every configuration parameter has a default. There is only one commandline flag argument that is required of the user to get things up and running. The user is expected to have a [huggingface user access token](https://huggingface.co/docs/hub/security-tokens) that will be used to upload the model post finetuning to HuggingFace hub."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: aaeeb883-dc2d-4bda-a1c7-9de085a83a20
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ddpo_trainer", "title": "Denoising Diffusion Policy Optimization", "tokens": 1924, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"aaeeb883-dc2d-4bda-a1c7-9de085a83a20\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"99e5bce9-55ed-41e1-8560-cc983baf3e9f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b0634b27587b9aebba82d04ac07e3dc86d3aadb8d744491d5e0c1e44640d14c3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1fcb2598-9dbd-4487-b72e-00c275088ae3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"a0457c9eb2cc0e8f74a6f09f476f94cdd055d5add3fd2a2a42fa4fb08b1423c3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d1d404a4-7638-490f-a840-476202080415\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d3277d8f7da1d77ed4893ab8eb3e1225af4ccf9db0d2612e44555f83658cd032\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2750, \"end_char_idx\": 6334, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "ref_doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "chroma:document": "For a more detailed look into the interface and the associated default implementation, go [here](https://github.com/lvwerra/trl/tree/main/trl/models/modeling_sd_base.py)\n\nNote that the default implementation has a LoRA implementation path and a non-LoRA based implementation path. The LoRA flag enabled by default and this can be turned off by passing in the flag to do so. LORA based training is faster and the LORA associated model hyperparameters responsible for model convergence aren't as finicky as non-LORA based training.\n\nAlso in addition, there is the expectation of providing a reward function and a prompt function. The reward function is used to evaluate the generated images  and the prompt function is used to generate the prompts that are used to generate the images.\n\n## Getting started with `examples/scripts/ddpo.py`\n\nThe `ddpo.py` script is a working example of using the `DDPO` trainer to finetune a Stable Diffusion model. This example explicitly configures a small subset of the overall parameters associated with the config object (`DDPOConfig`).\n\n**Note:** one A100 GPU is recommended to get this running. Anything below a A100 will not be able to run this example script and even if it does via relatively smaller sized parameters, the results will most likely be poor.\n\nAlmost every configuration parameter has a default. There is only one commandline flag argument that is required of the user to get things up and running. The user is expected to have a [huggingface user access token](https://huggingface.co/docs/hub/security-tokens) that will be used to upload the model post finetuning to HuggingFace hub. The following bash command is to be entered to get things running\n\n```batch\npython ddpo.py --hf_user_access_token <token>\n```\n\nTo obtain the documentation of `stable_diffusion_tuning.py`, please run `python stable_diffusion_tuning.py --help`\n\nThe following are things to keep in mind (The code checks this for you as well) in general while configuring the trainer (beyond the use case of using the example script)\n\n- The configurable sample batch size (`--ddpo_config.sample_batch_size=6`) should be greater than or equal to the configurable training batch size (`--ddpo_config.train_batch_size=3`)\n- The configurable sample batch size (`--ddpo_config.sample_batch_size=6`) must be divisible by the configurable train batch size (`--ddpo_config.train_batch_size=3`)\n- The configurable sample batch size (`--ddpo_config.sample_batch_size=6`) must be divisible by both the configurable gradient accumulation steps (`--ddpo_config.train_gradient_accumulation_steps=1`) and the configurable accelerator processes count \n\n## Setting up the image logging hook function\n\nExpect the function to be given a list of lists of the form\n```python\n[[image, prompt, prompt_metadata, rewards, reward_metadata], ...]\n\n```\nand `image`, `prompt`, `prompt_metadata`, `rewards`, `reward_metadata` are batched.\nThe last list in the lists of lists represents the last sample batch. You are likely to want to log this one\nWhile you are free to log however you want the use of `wandb` or `tensorboard` is recommended.\n\n### Key terms\n\n- `rewards` : The rewards/score is a numerical associated with the generated image and is key to steering the RL process\n- `reward_metadata` : The reward metadata is the metadata associated with the reward. Think of this as extra information payload delivered alongside the reward\n- `prompt` : The prompt is the text that is used to generate the image\n- `prompt_metadata` : The prompt metadata is the metadata associated with the prompt."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: d1d404a4-7638-490f-a840-476202080415
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/ddpo_trainer", "title": "Denoising Diffusion Policy Optimization", "tokens": 1924, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"d1d404a4-7638-490f-a840-476202080415\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"99e5bce9-55ed-41e1-8560-cc983baf3e9f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b0634b27587b9aebba82d04ac07e3dc86d3aadb8d744491d5e0c1e44640d14c3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"aaeeb883-dc2d-4bda-a1c7-9de085a83a20\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/ddpo_trainer\", \"title\": \"Denoising Diffusion Policy Optimization\", \"tokens\": 1924, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"8eb496b4b1fe88aa1ac2a81131ad36c47cb534dda0c56af5a10af5cf401f05b1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5677, \"end_char_idx\": 8514, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "ref_doc_id": "99e5bce9-55ed-41e1-8560-cc983baf3e9f", "chroma:document": "The last list in the lists of lists represents the last sample batch. You are likely to want to log this one\nWhile you are free to log however you want the use of `wandb` or `tensorboard` is recommended.\n\n### Key terms\n\n- `rewards` : The rewards/score is a numerical associated with the generated image and is key to steering the RL process\n- `reward_metadata` : The reward metadata is the metadata associated with the reward. Think of this as extra information payload delivered alongside the reward\n- `prompt` : The prompt is the text that is used to generate the image\n- `prompt_metadata` : The prompt metadata is the metadata associated with the prompt. A situation where this will not be empty is when the reward model comprises of a [`FLAVA`](https://huggingface.co/docs/transformers/model_doc/flava) setup where questions and ground answers (linked to the generated image) are expected with the generated image (See here: https://github.com/kvablack/ddpo-pytorch/blob/main/ddpo_pytorch/rewards.py#L45)\n- `image` : The image generated by the Stable Diffusion model\n\nExample code for logging sampled images with `wandb` is given below.\n\n```python\n# for logging these images to wandb\n\ndef image_outputs_hook(image_data, global_step, accelerate_logger):\n    # For the sake of this example, we only care about the last batch\n    # hence we extract the last element of the list\n    result = {}\n    images, prompts, _, rewards, _ = image_data[-1]\n    for i, image in enumerate(images):\n        pil = Image.fromarray(\n            (image.cpu().numpy().transpose(1, 2, 0) * 255).astype(np.uint8)\n        )\n        pil = pil.resize((256, 256))\n        result[f\"{prompts[i]:.25} | {rewards[i]:.2f}\"] = [pil]\n    accelerate_logger.log_images(\n        result,\n        step=global_step,\n    )\n\n```\n\n### Using the finetuned model\n\nAssuming you've done with all the epochs and have pushed up your model to the hub, you can use the finetuned model as follows\n\n```python\n\nimport torch\nfrom trl import DefaultDDPOStableDiffusionPipeline\n\npipeline = DefaultDDPOStableDiffusionPipeline(\"metric-space/ddpo-finetuned-sd-model\")\n\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n\n# memory optimization\npipeline.vae.to(device, torch.float16)\npipeline.text_encoder.to(device, torch.float16)\npipeline.unet.to(device, torch.float16)\n\nprompts = [\"squirrel\", \"crab\", \"starfish\", \"whale\",\"sponge\", \"plankton\"]\nresults = pipeline(prompts)\n\nfor prompt, image in zip(prompts,results.images):\n    image.save(f\"{prompt}.png\")\n\n```\n\n## Credits\n\nThis work is heavily influenced by the repo [here](https://github.com/kvablack/ddpo-pytorch) and the associated paper [Training Diffusion Models\nwith Reinforcement Learning by Kevin Black, Michael Janner, Yilan Du, Ilya Kostrikov, Sergey Levine](https://huggingface.co/papers/2305.13301)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 7a1029b0-cbdb-4daa-b83c-90ec8df17269
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"7a1029b0-cbdb-4daa-b83c-90ec8df17269\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"de5f4bb2-1631-47c0-a0c5-c53d477a6a05\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8f70e66203c4f55e2d6ccad5b507d3e450f92f044fbb3136637ee0b28ceaf6d9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2901, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "# Learning Tools (Experimental \ud83e\uddea)\n\nUsing Large Language Models (LLMs) with tools has been a popular topic recently with awesome works such as [ToolFormer](https://huggingface.co/papers/2302.04761) and [ToolBench](https://huggingface.co/papers/2305.16504). In TRL, we provide a simple example of how to teach LLM to use tools with reinforcement learning. \n\n\nHere's an overview of the scripts in the [trl repository](https://github.com/lvwerra/trl/tree/main/examples/research_projects/tools):\n\n| File | Description | \n|---|---| \n| [`calculator.py`](https://github.com/lvwerra/trl/blob/main/examples/research_projects/tools/calculator.py) | Script to train LLM to use a calculator with reinforcement learning. |\n| [`triviaqa.py`](https://github.com/lvwerra/trl/blob/main/examples/research_projects/tools/triviaqa.py) | Script to train LLM to use a wiki tool to answer questions. |\n| [`python_interpreter.py`](https://github.com/lvwerra/trl/blob/main/examples/research_projects/tools/python_interpreter.py) | Script to train LLM to use python interpreter to solve math puzzles. |\n\n<Tip warning={true}>\n\nNote that the scripts above rely heavily on the `TextEnvironment` API which is still under active development. The API may change in the future. Please see [`TextEnvironment`](text_environment) for the related docs.\n</Tip>\n\n\n## Learning to Use a Calculator\n\n\nThe rough idea is as follows:\n\n1. Load a tool such as [ybelkada/simple-calculator](https://huggingface.co/spaces/ybelkada/simple-calculator) that parse a text calculation like `\"14 + 34\"` and return the calulated number:\n    ```python\n    from transformers import AutoTokenizer, load_tool\n    tool = load_tool(\"ybelkada/simple-calculator\")\n    tool_fn = lambda text: str(round(float(tool(text)), 2))  # rounding to 2 decimal places\n    ```\n1. Define a reward function that returns a positive reward if the tool returns the correct answer. In the script we create a dummy reward function like `reward_fn = lambda x: 1`, but we override the rewards directly later.\n1. Create a prompt on how to use the tools\n    ```python\n    # system prompt\n    prompt = \"\"\"\\\n    What is 13.1-3?\n\n    <request><SimpleCalculatorTool>13.1-3<call>10.1<response>\n\n    Result=10.1<submit>\n\n    What is 4*3?\n\n    <request><SimpleCalculatorTool>4*3<call>12<response>\n\n    Result=12<submit>\n\n    What is 12.1+1?\n\n    <request><SimpleCalculatorTool>12.1+1<call>13.1<response>\n\n    Result=13.1<submit>\n\n    What is 12.1-20?\n\n    <request><SimpleCalculatorTool>12.1-20<call>-7.9<response>\n\n    Result=-7.9<submit>\"\"\"\n    ```\n3. Create a `trl.TextEnvironment` with the model \n    ```python\n    env = TextEnvironment(\n        model,\n        tokenizer,\n        {\"SimpleCalculatorTool\": tool_fn},\n        reward_fn,\n        prompt,\n        generation_kwargs=generation_kwargs,\n    )\n    ```\n4. Then generate some data such as `tasks = [\"\\n\\nWhat is 13.1-3?\", \"\\n\\nWhat is 4*3?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: de5f4bb2-1631-47c0-a0c5-c53d477a6a05
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"de5f4bb2-1631-47c0-a0c5-c53d477a6a05\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7a1029b0-cbdb-4daa-b83c-90ec8df17269\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c0190615f5aa7707d6a693419a261d1156c2c5497e0f9a6dda17a4b2a6f401ba\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"edb10951-3881-4ffb-93fe-9171d77bd9be\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b3b4053327fe804fe2a5f8ce897afbeaf26c2c5ae167e3cefc10d664e4dcfec1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1801, \"end_char_idx\": 4456, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "Define a reward function that returns a positive reward if the tool returns the correct answer. In the script we create a dummy reward function like `reward_fn = lambda x: 1`, but we override the rewards directly later.\n1. Create a prompt on how to use the tools\n    ```python\n    # system prompt\n    prompt = \"\"\"\\\n    What is 13.1-3?\n\n    <request><SimpleCalculatorTool>13.1-3<call>10.1<response>\n\n    Result=10.1<submit>\n\n    What is 4*3?\n\n    <request><SimpleCalculatorTool>4*3<call>12<response>\n\n    Result=12<submit>\n\n    What is 12.1+1?\n\n    <request><SimpleCalculatorTool>12.1+1<call>13.1<response>\n\n    Result=13.1<submit>\n\n    What is 12.1-20?\n\n    <request><SimpleCalculatorTool>12.1-20<call>-7.9<response>\n\n    Result=-7.9<submit>\"\"\"\n    ```\n3. Create a `trl.TextEnvironment` with the model \n    ```python\n    env = TextEnvironment(\n        model,\n        tokenizer,\n        {\"SimpleCalculatorTool\": tool_fn},\n        reward_fn,\n        prompt,\n        generation_kwargs=generation_kwargs,\n    )\n    ```\n4. Then generate some data such as `tasks = [\"\\n\\nWhat is 13.1-3?\", \"\\n\\nWhat is 4*3?\"]` and run the environment with `queries, responses, masks, rewards, histories = env.run(tasks)`. The environment will look for the `<call>` token in the prompt and append the tool output to the response; it will also return the mask associated with the response. You can further use the `histories` to visualize the interaction between the model and the tool; `histories[0].show_text()` will show the text with color-coded tool output and `histories[0].show_tokens(tokenizer)` will show visualize the tokens.\n    ![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/learning_tools.png)\n1. Finally, we can train the model with `train_stats = ppo_trainer.step(queries, responses, rewards, masks)`. The trainer will use the mask to ignore the tool output when computing the loss, make sure to pass that argument to `step`.\n\n## Experiment results\n\nWe trained a model with the above script for 10 random seeds. You can reproduce the run with the following command. Feel free to remove the `--slurm-*` arguments if you don't have access to a slurm cluster.\n\n```\nWANDB_TAGS=\"calculator_final\" python benchmark/benchmark.py \\\n    --command \"python examples/research_projects/tools/calculator.py\" \\\n    --num-seeds 10 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 8 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\nWe can then use [`openrlbenchmark`](https://github.com/openrlbenchmark/openrlbenchmark) which generates the following plot."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: edb10951-3881-4ffb-93fe-9171d77bd9be
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"edb10951-3881-4ffb-93fe-9171d77bd9be\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"de5f4bb2-1631-47c0-a0c5-c53d477a6a05\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"57ec603ce3562e21eca1dafee59aa3a9c37c00050b932b09b8e37bf15db5a62a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fb852ffa-77c4-45ab-bdd7-1172f5e2c382\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8cc2eeed4f8793afccbe0f141d73bd25a6c9ac3de160d49414c7333338bdbac2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2881, \"end_char_idx\": 5886, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "\", \"\\n\\nWhat is 4*3?\"]` and run the environment with `queries, responses, masks, rewards, histories = env.run(tasks)`. The environment will look for the `<call>` token in the prompt and append the tool output to the response; it will also return the mask associated with the response. You can further use the `histories` to visualize the interaction between the model and the tool; `histories[0].show_text()` will show the text with color-coded tool output and `histories[0].show_tokens(tokenizer)` will show visualize the tokens.\n    ![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/learning_tools.png)\n1. Finally, we can train the model with `train_stats = ppo_trainer.step(queries, responses, rewards, masks)`. The trainer will use the mask to ignore the tool output when computing the loss, make sure to pass that argument to `step`.\n\n## Experiment results\n\nWe trained a model with the above script for 10 random seeds. You can reproduce the run with the following command. Feel free to remove the `--slurm-*` arguments if you don't have access to a slurm cluster.\n\n```\nWANDB_TAGS=\"calculator_final\" python benchmark/benchmark.py \\\n    --command \"python examples/research_projects/tools/calculator.py\" \\\n    --num-seeds 10 \\\n    --start-seed 1 \\\n    --workers 10 \\\n    --slurm-gpus-per-task 1 \\\n    --slurm-ntasks 1 \\\n    --slurm-total-cpus 8 \\\n    --slurm-template-path benchmark/trl.slurm_template\n```\n\nWe can then use [`openrlbenchmark`](https://github.com/openrlbenchmark/openrlbenchmark) which generates the following plot.\n```\npython -m openrlbenchmark.rlops_multi_metrics \\\n    --filters '?we=openrlbenchmark&wpn=trl&xaxis=_step&ceik=trl_ppo_trainer_config.value.tracker_project_name&cen=trl_ppo_trainer_config.value.log_with&metrics=env/reward_mean&metrics=objective/kl' \\\n        'wandb?tag=calculator_final&cl=calculator_mask' \\\n    --env-ids trl \\\n    --check-empty-runs \\\n    --pc.ncols 2 \\\n    --pc.ncols-legend 1 \\\n    --output-filename static/0compare \\\n    --scan-history\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/learning_tools_chart.png)\n\nAs we can see, while 1-2 experiments crashed for some reason, most of the runs obtained near perfect proficiency in the calculator task.\n\n\n## (Early Experiments \ud83e\uddea): learning to use a wiki tool for question answering\n\nIn the [ToolFormer](https://huggingface.co/papers/2302.04761) paper, it shows an interesting use case that utilizes a Wikipedia Search tool to help answer questions. In this section, we attempt to perform similar experiments but uses RL instead to teach the model to use a wiki tool on the [TriviaQA](https://nlp.cs.washington.edu/triviaqa/) dataset.\n\n\n<Tip warning={true}>\n\n**Note that many settings are different so the results are not directly comparable.**\n</Tip>\n\n\n\n\n### Building a search index\n\nSince [ToolFormer](https://huggingface.co/papers/2302.04761) did not open source, we needed to first replicate the search index."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: fb852ffa-77c4-45ab-bdd7-1172f5e2c382
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"fb852ffa-77c4-45ab-bdd7-1172f5e2c382\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"edb10951-3881-4ffb-93fe-9171d77bd9be\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"ff0fa39dcb98df385ac1342432ce6a8a5e0f96e1f3d334e586c6bc54b2815199\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6f3ca07d-54c6-4969-ad96-882f684946d1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f231ea22bd0e80c9cd33cb7f7b42805ec9332b189a8628d103cd6aee4eec512f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4457, \"end_char_idx\": 7723, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "```\npython -m openrlbenchmark.rlops_multi_metrics \\\n    --filters '?we=openrlbenchmark&wpn=trl&xaxis=_step&ceik=trl_ppo_trainer_config.value.tracker_project_name&cen=trl_ppo_trainer_config.value.log_with&metrics=env/reward_mean&metrics=objective/kl' \\\n        'wandb?tag=calculator_final&cl=calculator_mask' \\\n    --env-ids trl \\\n    --check-empty-runs \\\n    --pc.ncols 2 \\\n    --pc.ncols-legend 1 \\\n    --output-filename static/0compare \\\n    --scan-history\n```\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/learning_tools_chart.png)\n\nAs we can see, while 1-2 experiments crashed for some reason, most of the runs obtained near perfect proficiency in the calculator task.\n\n\n## (Early Experiments \ud83e\uddea): learning to use a wiki tool for question answering\n\nIn the [ToolFormer](https://huggingface.co/papers/2302.04761) paper, it shows an interesting use case that utilizes a Wikipedia Search tool to help answer questions. In this section, we attempt to perform similar experiments but uses RL instead to teach the model to use a wiki tool on the [TriviaQA](https://nlp.cs.washington.edu/triviaqa/) dataset.\n\n\n<Tip warning={true}>\n\n**Note that many settings are different so the results are not directly comparable.**\n</Tip>\n\n\n\n\n### Building a search index\n\nSince [ToolFormer](https://huggingface.co/papers/2302.04761) did not open source, we needed to first replicate the search index. It is mentioned in their paper that the authors built the search index using a BM25 retriever that indexes the Wikipedia dump from [KILT](https://github.com/facebookresearch/KILT)\n\nFortunately, [`pyserini`](https://github.com/castorini/pyserini) already implements the BM25 retriever and provides a prebuilt index for the KILT Wikipedia dump. We can use the following code to search the index.\n\n```python\nfrom pyserini.search.lucene import LuceneSearcher\nimport json\nsearcher = LuceneSearcher.from_prebuilt_index('wikipedia-kilt-doc')\ndef search(query):\n    hits = searcher.search(query, k=1)\n    hit = hits[0]\n    contents = json.loads(hit.raw)['contents']\n    return contents\nprint(search(\"tennis racket\"))\n```\n```\nRacket (sports equipment)\nA racket or racquet is a sports implement consisting of a handled frame with an open hoop across which a network of strings or catgut is stretched tightly. It is used for striking a ball or shuttlecock in games such as squash, tennis, racquetball, and badminton. Collectively, these games are known as racket sports. Racket design and manufacturing has changed considerably over the centuries.\n\nThe frame of rackets for all sports was traditionally made of solid wood (later laminated wood) and the strings of animal intestine known as catgut. The traditional racket size was limited by the strength and weight of the wooden frame which had to be strong enough to hold the strings and stiff enough to hit the ball or shuttle. Manufacturers started adding non-wood laminates to wood rackets to improve stiffness. Non-wood rackets were made first of steel, then of aluminum, and then carbon fiber composites. Wood is still used for real tennis, rackets, and xare. Most rackets are now made of composite materials including carbon fiber or fiberglass, metals such as titanium alloys, or ceramics."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 6f3ca07d-54c6-4969-ad96-882f684946d1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"6f3ca07d-54c6-4969-ad96-882f684946d1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fb852ffa-77c4-45ab-bdd7-1172f5e2c382\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4f9c40eec3f92adc63b369cdfd26d130e9771c688ce0e69c32faf0895316b0c5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5ecdf0a1-80be-4ce6-aa3b-b8331993d9a9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"55407e0b5cee7c6ce310ba81fc6e956cdc56fbcbfb362ff8b5b384ae9758fbd4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6230, \"end_char_idx\": 9273, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "We can use the following code to search the index.\n\n```python\nfrom pyserini.search.lucene import LuceneSearcher\nimport json\nsearcher = LuceneSearcher.from_prebuilt_index('wikipedia-kilt-doc')\ndef search(query):\n    hits = searcher.search(query, k=1)\n    hit = hits[0]\n    contents = json.loads(hit.raw)['contents']\n    return contents\nprint(search(\"tennis racket\"))\n```\n```\nRacket (sports equipment)\nA racket or racquet is a sports implement consisting of a handled frame with an open hoop across which a network of strings or catgut is stretched tightly. It is used for striking a ball or shuttlecock in games such as squash, tennis, racquetball, and badminton. Collectively, these games are known as racket sports. Racket design and manufacturing has changed considerably over the centuries.\n\nThe frame of rackets for all sports was traditionally made of solid wood (later laminated wood) and the strings of animal intestine known as catgut. The traditional racket size was limited by the strength and weight of the wooden frame which had to be strong enough to hold the strings and stiff enough to hit the ball or shuttle. Manufacturers started adding non-wood laminates to wood rackets to improve stiffness. Non-wood rackets were made first of steel, then of aluminum, and then carbon fiber composites. Wood is still used for real tennis, rackets, and xare. Most rackets are now made of composite materials including carbon fiber or fiberglass, metals such as titanium alloys, or ceramics.\n...\n```\n\nWe then basically deployed this snippet as a Hugging Face space [here](https://huggingface.co/spaces/vwxyzjn/pyserini-wikipedia-kilt-doc), so that we can use the space as a `transformers.Tool` later.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/pyserini.png)\n\n### Experiment settings\n\nWe use the following settings:\n\n* use the `bigcode/starcoderbase` model as the base model\n* use the `pyserini-wikipedia-kilt-doc` space as the wiki tool and only uses the first paragrahs of the search result, allowing the `TextEnvironment` to obtain at most `max_tool_reponse=400` response tokens from the tool.\n* test if the response contain the answer string, if so, give a reward of 1, otherwise, give a reward of 0.\n    * notice this is a simplified evaluation criteria. In [ToolFormer](https://huggingface.co/papers/2302.04761), the authors checks if the first 20 words of the response contain the correct answer.\n* used the following prompt that demonstrates the usage of the wiki tool.\n```python\nprompt = \"\"\"\\\nAnswer the following question:\n\nQ: In which branch of the arts is Patricia Neary famous?\nA: Ballets\nA2: <request><Wiki>Patricia Neary<call>Patricia Neary (born October 27, 1942) is an American ballerina, choreographer and ballet director, who has been particularly active in Switzerland. She has also been a highly successful ambassador for the Balanchine Trust, bringing George Balanchine's ballets to 60 cities around the globe.<response>\nResult=Ballets<submit>\n\nQ: Who won Super Bowl XX?"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 5ecdf0a1-80be-4ce6-aa3b-b8331993d9a9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"5ecdf0a1-80be-4ce6-aa3b-b8331993d9a9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6f3ca07d-54c6-4969-ad96-882f684946d1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"57024b26e3dd167952aec84abe5642db2c914b963ba35c05249bfc317493e14d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ce3d8779-7e92-4ed4-a0d7-da67bd20f093\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e6c11774c22a0a20eb72eec2ab2a3b95ea90e5c6392755b85cf818cf15920cd2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7724, \"end_char_idx\": 10387, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "...\n```\n\nWe then basically deployed this snippet as a Hugging Face space [here](https://huggingface.co/spaces/vwxyzjn/pyserini-wikipedia-kilt-doc), so that we can use the space as a `transformers.Tool` later.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/pyserini.png)\n\n### Experiment settings\n\nWe use the following settings:\n\n* use the `bigcode/starcoderbase` model as the base model\n* use the `pyserini-wikipedia-kilt-doc` space as the wiki tool and only uses the first paragrahs of the search result, allowing the `TextEnvironment` to obtain at most `max_tool_reponse=400` response tokens from the tool.\n* test if the response contain the answer string, if so, give a reward of 1, otherwise, give a reward of 0.\n    * notice this is a simplified evaluation criteria. In [ToolFormer](https://huggingface.co/papers/2302.04761), the authors checks if the first 20 words of the response contain the correct answer.\n* used the following prompt that demonstrates the usage of the wiki tool.\n```python\nprompt = \"\"\"\\\nAnswer the following question:\n\nQ: In which branch of the arts is Patricia Neary famous?\nA: Ballets\nA2: <request><Wiki>Patricia Neary<call>Patricia Neary (born October 27, 1942) is an American ballerina, choreographer and ballet director, who has been particularly active in Switzerland. She has also been a highly successful ambassador for the Balanchine Trust, bringing George Balanchine's ballets to 60 cities around the globe.<response>\nResult=Ballets<submit>\n\nQ: Who won Super Bowl XX?\nA: Chicago Bears\nA2: <request><Wiki>Super Bowl XX<call>Super Bowl XX was an American football game between the National Football Conference (NFC) champion Chicago Bears and the American Football Conference (AFC) champion New England Patriots to decide the National Football League (NFL) champion for the 1985 season. The Bears defeated the Patriots by the score of 46\u201310, capturing their first NFL championship (and Chicago's first overall sports victory) since 1963, three years prior to the birth of the Super Bowl. Super Bowl XX was played on January 26, 1986 at the Louisiana Superdome in New Orleans.<response>\nResult=Chicago Bears<submit>\n\nQ: \"\"\"\n```\n\n\n### Result and Discussion\n\n\nOur experiments show that the agent can learn to use the wiki tool to answer questions. The learning curves would go up mostly, but one of the experiment did crash.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/triviaqa_learning_curves.png)\n\nWandb report is [here](https://wandb.ai/costa-huang/cleanRL/reports/TriviaQA-Final-Experiments--Vmlldzo1MjY0ODk5) for further inspection."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: ce3d8779-7e92-4ed4-a0d7-da67bd20f093
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"ce3d8779-7e92-4ed4-a0d7-da67bd20f093\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5ecdf0a1-80be-4ce6-aa3b-b8331993d9a9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"12f48a280adae35d76c7a9c91caa426788b0d4f48466e5405528f1cef4bc9582\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b1ac48ce-6e48-4e48-bac6-9915fe7c3598\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"48bc5bd1ce601f78dd67718b4cbe607d608b5817c167686d43c4241dfe5f3721\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8759, \"end_char_idx\": 11178, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "```python\nprompt = \"\"\"\\\nAnswer the following question:\n\nQ: In which branch of the arts is Patricia Neary famous?\nA: Ballets\nA2: <request><Wiki>Patricia Neary<call>Patricia Neary (born October 27, 1942) is an American ballerina, choreographer and ballet director, who has been particularly active in Switzerland. She has also been a highly successful ambassador for the Balanchine Trust, bringing George Balanchine's ballets to 60 cities around the globe.<response>\nResult=Ballets<submit>\n\nQ: Who won Super Bowl XX?\nA: Chicago Bears\nA2: <request><Wiki>Super Bowl XX<call>Super Bowl XX was an American football game between the National Football Conference (NFC) champion Chicago Bears and the American Football Conference (AFC) champion New England Patriots to decide the National Football League (NFL) champion for the 1985 season. The Bears defeated the Patriots by the score of 46\u201310, capturing their first NFL championship (and Chicago's first overall sports victory) since 1963, three years prior to the birth of the Super Bowl. Super Bowl XX was played on January 26, 1986 at the Louisiana Superdome in New Orleans.<response>\nResult=Chicago Bears<submit>\n\nQ: \"\"\"\n```\n\n\n### Result and Discussion\n\n\nOur experiments show that the agent can learn to use the wiki tool to answer questions. The learning curves would go up mostly, but one of the experiment did crash.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/triviaqa_learning_curves.png)\n\nWandb report is [here](https://wandb.ai/costa-huang/cleanRL/reports/TriviaQA-Final-Experiments--Vmlldzo1MjY0ODk5) for further inspection.\n\n\nNote that the correct rate of the trained model is on the low end, which could be due to the following reasons:\n\n* **incorrect searches:** When given the question `\"What is Bruce Willis' real first name?\"` if the model searches for `Bruce Willis`, our wiki tool returns \"Patrick Poivey (born 18 February 1948) is a French actor. He is especially known for his voice: he is the French dub voice of Bruce Willis since 1988.` But a correct search should be `Walter Bruce Willis (born March 19, 1955) is an American former actor. He achieved fame with a leading role on the comedy-drama series Moonlighting (1985\u20131989) and appeared in over a hundred films, gaining recognition as an action hero after his portrayal of John McClane in the Die Hard franchise (1988\u20132013) and other roles.[1][2]\""}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: b1ac48ce-6e48-4e48-bac6-9915fe7c3598
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"b1ac48ce-6e48-4e48-bac6-9915fe7c3598\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ce3d8779-7e92-4ed4-a0d7-da67bd20f093\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"4eb582ffccbe55eb5c19a312041f01b8062025ac8492336c8cab0313bf282cd7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1df4cdcd-abc2-4784-9b7a-ad7ce2eb6e86\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3fe5b2363d6d916a24180639b8ed9049b2e047ad7771d6bba1ea2bf9ad8ce7e8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9591, \"end_char_idx\": 12863, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "The Bears defeated the Patriots by the score of 46\u201310, capturing their first NFL championship (and Chicago's first overall sports victory) since 1963, three years prior to the birth of the Super Bowl. Super Bowl XX was played on January 26, 1986 at the Louisiana Superdome in New Orleans.<response>\nResult=Chicago Bears<submit>\n\nQ: \"\"\"\n```\n\n\n### Result and Discussion\n\n\nOur experiments show that the agent can learn to use the wiki tool to answer questions. The learning curves would go up mostly, but one of the experiment did crash.\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/triviaqa_learning_curves.png)\n\nWandb report is [here](https://wandb.ai/costa-huang/cleanRL/reports/TriviaQA-Final-Experiments--Vmlldzo1MjY0ODk5) for further inspection.\n\n\nNote that the correct rate of the trained model is on the low end, which could be due to the following reasons:\n\n* **incorrect searches:** When given the question `\"What is Bruce Willis' real first name?\"` if the model searches for `Bruce Willis`, our wiki tool returns \"Patrick Poivey (born 18 February 1948) is a French actor. He is especially known for his voice: he is the French dub voice of Bruce Willis since 1988.` But a correct search should be `Walter Bruce Willis (born March 19, 1955) is an American former actor. He achieved fame with a leading role on the comedy-drama series Moonlighting (1985\u20131989) and appeared in over a hundred films, gaining recognition as an action hero after his portrayal of John McClane in the Die Hard franchise (1988\u20132013) and other roles.[1][2]\"\n\n\n    ![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/real_first_name.png)\n\n* **unnecessarily long response**: The wiki tool by default sometimes output very long sequences. E.g., when the wiki tool searches for \"Brown Act\"\n    * Our wiki tool returns \"The Ralph M. Brown Act, located at California Government Code 54950 \"et seq.\", is an act of the California State Legislature, authored by Assemblymember Ralph M. Brown and passed in 1953, that guarantees the public's right to attend and participate in meetings of local legislative bodies.\"\n    * [ToolFormer](https://huggingface.co/papers/2302.04761)'s wiki tool returns \"The Ralph M. Brown Act is an act of the California State Legislature that guarantees the public's right to attend and participate in meetings of local legislative bodies.\" which is more succinct.\n\n    ![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/brown_act.png)\n\n\n## (Early Experiments \ud83e\uddea): solving math puzzles with python interpreter\n\nIn this section, we attempt to teach the model to use a python interpreter to solve math puzzles. The rough idea is to give the agent a prompt like the following:\n\n```python\nprompt = \"\"\"\\\nExample of using a Python API to solve math questions.\n\nQ: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n\n<request><PythonInterpreter>\ndef solution():\n    money_initial = 23\n    bagels = 5\n    bagel_cost = 3\n    money_spent = bagels * bagel_cost\n    money_left = money_initial - money_spent\n    result = money_left\n    return result\nprint(solution())\n<call>72<response>\n\nResult = 72 <submit>\n\nQ: \"\"\"\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 1df4cdcd-abc2-4784-9b7a-ad7ce2eb6e86
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/learning_tools", "title": "Learning Tools (Experimental \ud83e\uddea)", "tokens": 3184, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"1df4cdcd-abc2-4784-9b7a-ad7ce2eb6e86\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2831b0af-5745-4c78-b3ea-4ac5967d9d4c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"add7c31d2128098622a1fde39ebd09c9501bf5e0f8bb4fb626ae0619ea2aac21\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b1ac48ce-6e48-4e48-bac6-9915fe7c3598\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/learning_tools\", \"title\": \"Learning Tools (Experimental \\ud83e\\uddea)\", \"tokens\": 3184, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b6fdb873d76949d196ee12ea1d87609158f4301018ef07fde1ff7a5f7b06b73f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11185, \"end_char_idx\": 13069, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "ref_doc_id": "2831b0af-5745-4c78-b3ea-4ac5967d9d4c", "chroma:document": "![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/real_first_name.png)\n\n* **unnecessarily long response**: The wiki tool by default sometimes output very long sequences. E.g., when the wiki tool searches for \"Brown Act\"\n    * Our wiki tool returns \"The Ralph M. Brown Act, located at California Government Code 54950 \"et seq.\", is an act of the California State Legislature, authored by Assemblymember Ralph M. Brown and passed in 1953, that guarantees the public's right to attend and participate in meetings of local legislative bodies.\"\n    * [ToolFormer](https://huggingface.co/papers/2302.04761)'s wiki tool returns \"The Ralph M. Brown Act is an act of the California State Legislature that guarantees the public's right to attend and participate in meetings of local legislative bodies.\" which is more succinct.\n\n    ![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/brown_act.png)\n\n\n## (Early Experiments \ud83e\uddea): solving math puzzles with python interpreter\n\nIn this section, we attempt to teach the model to use a python interpreter to solve math puzzles. The rough idea is to give the agent a prompt like the following:\n\n```python\nprompt = \"\"\"\\\nExample of using a Python API to solve math questions.\n\nQ: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n\n<request><PythonInterpreter>\ndef solution():\n    money_initial = 23\n    bagels = 5\n    bagel_cost = 3\n    money_spent = bagels * bagel_cost\n    money_left = money_initial - money_spent\n    result = money_left\n    return result\nprint(solution())\n<call>72<response>\n\nResult = 72 <submit>\n\nQ: \"\"\"\n```\n\n\nTraining experiment can be found at https://wandb.ai/lvwerra/trl-gsm8k/runs/a5odv01y\n\n![](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/images/gms8k_learning_curve.png)"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 83f7bd4b-be41-4745-8865-24b0c8860b80
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/example_overview", "title": "Examples", "tokens": 1870, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"83f7bd4b-be41-4745-8865-24b0c8860b80\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4395e4c9-16cd-4654-b349-69caa9a0f3bb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"70442542ee78bccf060063edae6104b824db40f531ba844b131f73c590a844fc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0e826995-dbff-41ec-9e66-7d873be183ab\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d96620a474131de3e63cec141f7688dc8156f79af08af4f007a604aee215355b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 4649, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "ref_doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "chroma:document": "# Examples\n\n\n## Introduction\n\nThe examples should work in any of the following settings (with the same script):\n   - single GPU\n   - multi GPUS (using PyTorch distributed mode)\n   - multi GPUS (using DeepSpeed ZeRO-Offload stages 1, 2, & 3)\n   - fp16 (mixed-precision), fp32 (normal precision), or bf16 (bfloat16 precision)\n\nTo run it in each of these various modes, first initialize the accelerate\nconfiguration with `accelerate config`\n\n**NOTE to train with a 4-bit or 8-bit model**, please run\n\n```bash\npip install --upgrade trl[quantization]\n```\n\n\n## Accelerate Config\nFor all the examples, you'll need to generate a \ud83e\udd17 Accelerate config file with:\n\n```shell\naccelerate config # will prompt you to define the training configuration\n```\n\nThen, it is encouraged to launch jobs with `accelerate launch`!\n\n\n# Maintained Examples\n\n\n\n| File                                                                                                                          | Description                                                                                                                                                                                                                                                                                                     |\n| ----------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| [`examples/scripts/alignprop.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/alignprop.py)                 | This script shows how to use the [`AlignPropTrainer`] to fine-tune a diffusion model.                                                                                                                                                                                                                           |\n| [`examples/scripts/bco.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/bco.py)                             | This script shows how to use the [`KTOTrainer`] with the BCO loss to fine-tune a model to increase instruction-following, truthfulness, honesty and helpfulness using the [openbmb/UltraFeedback](https://huggingface.co/datasets/openbmb/UltraFeedback) dataset.                                               |\n| [`examples/scripts/chat.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/chat.py)                           | This script allows you to load and use a model as a chatbot.                                                                                                                                                                                                                                                    |\n| [`examples/scripts/cpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/cpo.py)                             | This script shows how to use the [`CPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset.                                                                                                         |\n| [`examples/scripts/ddpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ddpo.py)                           | This script shows how to use the [`DDPOTrainer`] to fine-tune a stable diffusion model using reinforcement learning.                                                                                                                                                                                            |\n| [`examples/scripts/dpo_visual.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo_visual.py)               | This script shows how to use the [`DPOTrainer`] to fine-tune a Vision Language Model to reduce hallucinations using the [openbmb/RLAIF-V-Dataset](https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset) dataset.                                                                                             |\n| [`examples/scripts/dpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo.py)                             | This script shows how to use the [`DPOTrainer`] to fine-tune a stable to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 0e826995-dbff-41ec-9e66-7d873be183ab
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/example_overview", "title": "Examples", "tokens": 1870, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"0e826995-dbff-41ec-9e66-7d873be183ab\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4395e4c9-16cd-4654-b349-69caa9a0f3bb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"70442542ee78bccf060063edae6104b824db40f531ba844b131f73c590a844fc\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"83f7bd4b-be41-4745-8865-24b0c8860b80\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"90b10d8adfedd9a59629d47d5014938330b04a649043ebe81b2aa386cdd2b83f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ee15f58a-8c16-4526-a544-3cd1aa661104\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"78c4ab59806373fe09fa344d372c911b6b9c5ebe633baa222f86a85d4715aa63\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2573, \"end_char_idx\": 7168, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "ref_doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "chroma:document": "|\n| [`examples/scripts/chat.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/chat.py)                           | This script allows you to load and use a model as a chatbot.                                                                                                                                                                                                                                                    |\n| [`examples/scripts/cpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/cpo.py)                             | This script shows how to use the [`CPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset.                                                                                                         |\n| [`examples/scripts/ddpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ddpo.py)                           | This script shows how to use the [`DDPOTrainer`] to fine-tune a stable diffusion model using reinforcement learning.                                                                                                                                                                                            |\n| [`examples/scripts/dpo_visual.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo_visual.py)               | This script shows how to use the [`DPOTrainer`] to fine-tune a Vision Language Model to reduce hallucinations using the [openbmb/RLAIF-V-Dataset](https://huggingface.co/datasets/openbmb/RLAIF-V-Dataset) dataset.                                                                                             |\n| [`examples/scripts/dpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/dpo.py)                             | This script shows how to use the [`DPOTrainer`] to fine-tune a stable to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset.                                                                                                        |\n| [`examples/scripts/kto.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/kto.py)                             | This script shows how to use the [`KTOTrainer`] to fine-tune a model.                                                                                                                                                                                                                                           |\n| [`examples/scripts/orpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/orpo.py)                           | This script shows how to use the [`ORPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset.                                                                                                        |\n| [`examples/scripts/ppo_multi_adapter.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo_multi_adapter.py) | This script shows how to use the [`PPOTrainer`] to train a single base model with multiple adapters. Requires you to run the example script with the reward model training beforehand.                                                                                                                          |\n| [`examples/scripts/ppo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo.py)                             | This script shows how to use the [`PPOTrainer`] to fine-tune a sentiment analysis model using [IMDB dataset](https://huggingface.co/datasets/stanfordnlp/imdb).                                                                                                                                                 |\n| [`examples/scripts/reward_modeling.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/reward_modeling.py)     | This script shows how to use the [`RewardTrainer`] to train a reward model on your own dataset.                                                                                                                                                                                                                 |\n| [`examples/scripts/sft.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/sft.py)                             | This script shows how to use the [`SFTTrainer`] to fine-tune a model or adapters into a target dataset."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: ee15f58a-8c16-4526-a544-3cd1aa661104
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/example_overview", "title": "Examples", "tokens": 1870, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"ee15f58a-8c16-4526-a544-3cd1aa661104\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4395e4c9-16cd-4654-b349-69caa9a0f3bb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"70442542ee78bccf060063edae6104b824db40f531ba844b131f73c590a844fc\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0e826995-dbff-41ec-9e66-7d873be183ab\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"aefa7ea1080dec8b8dd79164a08fa6375bfbae761e408e753d8b33f5509aaf2c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4173489b-0db5-489b-803a-85c96e4e8379\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2cabe4b472fecae894dfca552da5299ddbcb8a1d668f34049b33b3a0c295cd7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4753, \"end_char_idx\": 9179, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "ref_doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "chroma:document": "|\n| [`examples/scripts/kto.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/kto.py)                             | This script shows how to use the [`KTOTrainer`] to fine-tune a model.                                                                                                                                                                                                                                           |\n| [`examples/scripts/orpo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/orpo.py)                           | This script shows how to use the [`ORPOTrainer`] to fine-tune a model to increase helpfulness and harmlessness using the [Anthropic/hh-rlhf](https://huggingface.co/datasets/Anthropic/hh-rlhf) dataset.                                                                                                        |\n| [`examples/scripts/ppo_multi_adapter.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo_multi_adapter.py) | This script shows how to use the [`PPOTrainer`] to train a single base model with multiple adapters. Requires you to run the example script with the reward model training beforehand.                                                                                                                          |\n| [`examples/scripts/ppo.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/ppo.py)                             | This script shows how to use the [`PPOTrainer`] to fine-tune a sentiment analysis model using [IMDB dataset](https://huggingface.co/datasets/stanfordnlp/imdb).                                                                                                                                                 |\n| [`examples/scripts/reward_modeling.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/reward_modeling.py)     | This script shows how to use the [`RewardTrainer`] to train a reward model on your own dataset.                                                                                                                                                                                                                 |\n| [`examples/scripts/sft.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/sft.py)                             | This script shows how to use the [`SFTTrainer`] to fine-tune a model or adapters into a target dataset.                                                                                                                                                                                                         |\n| [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py)               | This script shows how to use the [`SFTTrainer`] to fine-tune a Vision Language Model in a chat setting. The script has only been tested on a [LLaVA 1.5]([llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf)) model so users may see unexpected behaviour in other model architectures. |\n\nHere are also some easier-to-run colab notebooks that you can use to get started with TRL:\n\n| File                                                                                                                              | Description                                                                                                             |\n| --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |\n| [`examples/notebooks/best_of_n.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/best_of_n.ipynb)           | This notebook demonstrates how to use the \"Best of N\" sampling strategy using TRL when fine-tuning your model with PPO. |\n| [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-sentiment.ipynb) | This notebook demonstrates how to reproduce the GPT2 imdb sentiment tuning example on a jupyter notebook.               |\n| [`examples/notebooks/gpt2-control.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-control.ipynb)     | This notebook demonstrates how to reproduce the GPT2 sentiment control example on a jupyter notebook.                   |"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 4173489b-0db5-489b-803a-85c96e4e8379
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/example_overview", "title": "Examples", "tokens": 1870, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"4173489b-0db5-489b-803a-85c96e4e8379\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4395e4c9-16cd-4654-b349-69caa9a0f3bb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"70442542ee78bccf060063edae6104b824db40f531ba844b131f73c590a844fc\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ee15f58a-8c16-4526-a544-3cd1aa661104\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/example_overview\", \"title\": \"Examples\", \"tokens\": 1870, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c19867aa21a4c093dd36d9af0f579807c2c5263a9052ab31b3b7580ed5b3e9cc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7369, \"end_char_idx\": 10800, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "ref_doc_id": "4395e4c9-16cd-4654-b349-69caa9a0f3bb", "chroma:document": "|\n| [`examples/scripts/vsft_llava.py`](https://github.com/huggingface/trl/blob/main/examples/scripts/vsft_llava.py)               | This script shows how to use the [`SFTTrainer`] to fine-tune a Vision Language Model in a chat setting. The script has only been tested on a [LLaVA 1.5]([llava-hf/llava-1.5-7b-hf](https://huggingface.co/llava-hf/llava-1.5-7b-hf)) model so users may see unexpected behaviour in other model architectures. |\n\nHere are also some easier-to-run colab notebooks that you can use to get started with TRL:\n\n| File                                                                                                                              | Description                                                                                                             |\n| --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |\n| [`examples/notebooks/best_of_n.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/best_of_n.ipynb)           | This notebook demonstrates how to use the \"Best of N\" sampling strategy using TRL when fine-tuning your model with PPO. |\n| [`examples/notebooks/gpt2-sentiment.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-sentiment.ipynb) | This notebook demonstrates how to reproduce the GPT2 imdb sentiment tuning example on a jupyter notebook.               |\n| [`examples/notebooks/gpt2-control.ipynb`](https://github.com/huggingface/trl/tree/main/examples/notebooks/gpt2-control.ipynb)     | This notebook demonstrates how to reproduce the GPT2 sentiment control example on a jupyter notebook.                   |\n\n\nWe also have some other examples that are less maintained but can be used as a reference:\n1. **[research_projects](https://github.com/huggingface/trl/tree/main/examples/research_projects)**: Check out this folder to find the scripts used for some research projects that used TRL (LM de-toxification, Stack-Llama, etc.)\n\n\n## Distributed training\n\nAll of the scripts can be run on multiple GPUs by providing the path of an \ud83e\udd17 Accelerate config file when calling `accelerate launch`. To launch one of them on one or multiple GPUs, run the following command (swapping `{NUM_GPUS}` with the number of GPUs in your machine and `--all_arguments_of_the_script` with your arguments.)\n\n```shell\naccelerate launch --config_file=examples/accelerate_configs/multi_gpu.yaml --num_processes {NUM_GPUS} path_to_script.py --all_arguments_of_the_script\n```\n\nYou can also adjust the parameters of the \ud83e\udd17 Accelerate config file to suit your needs (e.g. training in mixed precision).\n\n### Distributed training with DeepSpeed\n\nMost of the scripts can be run on multiple GPUs together with DeepSpeed ZeRO-{1,2,3} for efficient sharding of the optimizer states, gradients, and model weights. To do so, run following command (swapping `{NUM_GPUS}` with the number of GPUs in your machine, `--all_arguments_of_the_script` with your arguments, and `--deepspeed_config` with the path to the DeepSpeed config file such as `examples/deepspeed_configs/deepspeed_zero1.yaml`):\n\n```shell\naccelerate launch --config_file=examples/accelerate_configs/deepspeed_zero{1,2,3}.yaml --num_processes {NUM_GPUS} path_to_script.py --all_arguments_of_the_script\n```"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 2c57f318-2dcd-4ba5-b408-1f09c2ebfdad
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/using_llama_models", "title": "Using LLaMA models with TRL", "tokens": 2435, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"2c57f318-2dcd-4ba5-b408-1f09c2ebfdad\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93a491eb-cddf-48d0-82d2-52bc28a0bdce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"259be3a8f639379b43879cb2f6ee34ff3313ac5605649be4013eb4b9921baa51\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"663c0a58-3176-4584-97e1-726d7fbb099a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"82b6f055b202a4c1e9c691617f9f1a07eb207cfc8e01147ec71ce5b624f84a39\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3106, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "ref_doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "chroma:document": "# Using LLaMA models with TRL\n\nWe've begun rolling out examples to use Meta's LLaMA models in `trl` (see [Meta's LLaMA release](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/) for the original LLaMA model).\n\n## Efficient training strategies\n\nEven training the smallest LLaMA model requires an enormous amount of memory. Some quick math: in bf16, every parameter uses 2 bytes (in fp32 4 bytes) in addition to 8 bytes used, e.g., in the Adam optimizer (see the [performance docs](https://huggingface.co/docs/transformers/perf_train_gpu_one#optimizer) in Transformers for more info). So a 7B parameter model would use `(2+8)*7B=70GB` just to fit in memory and would likely need more when you compute intermediate values such as attention scores. So you couldn\u2019t train the model even on a single 80GB A100 like that. You can use some tricks, like more efficient optimizers of half-precision training, to squeeze a bit more into memory, but you\u2019ll run out sooner or later.\n\nAnother option is to use Parameter-Efficient Fine-Tuning (PEFT) techniques, such as the [`peft`](https://github.com/huggingface/peft) library, which can perform low-rank adaptation (LoRA) on a model loaded in 8-bit.\nFor more on `peft` + `trl`, see the [docs](https://huggingface.co/docs/trl/sentiment_tuning_peft).\n\nLoading the model in 8bit reduces the memory footprint drastically since you only need one byte per parameter for the weights (e.g. 7B LlaMa is 7GB in memory).\nInstead of training the original weights directly, LoRA adds small adapter layers on top of some specific layers (usually the attention layers); thus, the number of trainable parameters is drastically reduced.\n\nIn this scenario, a rule of thumb is to allocate ~1.2-1.4GB per billion parameters (depending on the batch size and sequence length) to fit the entire fine-tuning setup.\nThis enables fine-tuning larger models (up to 50-60B scale models on a NVIDIA A100 80GB) at low cost.\n\nNow we can fit very large models into a single GPU, but the training might still be very slow.\nThe simplest strategy in this scenario is data parallelism: we replicate the same training setup into separate GPUs and pass different batches to each GPU.\nWith this, you can parallelize the forward/backward passes of the model and scale with the number of GPUs.\n\n![chapter10_ddp.png](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/blog/stackllama/chapter10_ddp.png)\n\nWe use either the `transformers.Trainer` or `accelerate`, which both support data parallelism without any code changes, by simply passing arguments when calling the scripts with `torchrun` or `accelerate launch`. The following runs a training script with 8 GPUs on a single machine with `accelerate` and `torchrun`, respectively.\n\n```bash\naccelerate launch --multi_gpu --num_machines 1  --num_processes 8 my_accelerate_script.py\ntorchrun --nnodes 1  --nproc_per_node 8 my_torch_script.py\n```\n\n## Supervised fine-tuning\n\nBefore we start training reward models and tuning our model with RL, it helps if the model is already good in the domain we are interested in."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 663c0a58-3176-4584-97e1-726d7fbb099a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/using_llama_models", "title": "Using LLaMA models with TRL", "tokens": 2435, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"663c0a58-3176-4584-97e1-726d7fbb099a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93a491eb-cddf-48d0-82d2-52bc28a0bdce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"259be3a8f639379b43879cb2f6ee34ff3313ac5605649be4013eb4b9921baa51\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2c57f318-2dcd-4ba5-b408-1f09c2ebfdad\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"de2be5ddfec77d28800086297f51cb99e85beed42c1cbe3894a73f9794b2cf7b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"99ec0bee-4a71-4412-8a22-52c68e33f7ca\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d85f798857dfff6e4be0f60acb308747f0486a6396ce818d6dc9e9acd9b70a61\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1435, \"end_char_idx\": 4853, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "ref_doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "chroma:document": "7B LlaMa is 7GB in memory).\nInstead of training the original weights directly, LoRA adds small adapter layers on top of some specific layers (usually the attention layers); thus, the number of trainable parameters is drastically reduced.\n\nIn this scenario, a rule of thumb is to allocate ~1.2-1.4GB per billion parameters (depending on the batch size and sequence length) to fit the entire fine-tuning setup.\nThis enables fine-tuning larger models (up to 50-60B scale models on a NVIDIA A100 80GB) at low cost.\n\nNow we can fit very large models into a single GPU, but the training might still be very slow.\nThe simplest strategy in this scenario is data parallelism: we replicate the same training setup into separate GPUs and pass different batches to each GPU.\nWith this, you can parallelize the forward/backward passes of the model and scale with the number of GPUs.\n\n![chapter10_ddp.png](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/blog/stackllama/chapter10_ddp.png)\n\nWe use either the `transformers.Trainer` or `accelerate`, which both support data parallelism without any code changes, by simply passing arguments when calling the scripts with `torchrun` or `accelerate launch`. The following runs a training script with 8 GPUs on a single machine with `accelerate` and `torchrun`, respectively.\n\n```bash\naccelerate launch --multi_gpu --num_machines 1  --num_processes 8 my_accelerate_script.py\ntorchrun --nnodes 1  --nproc_per_node 8 my_torch_script.py\n```\n\n## Supervised fine-tuning\n\nBefore we start training reward models and tuning our model with RL, it helps if the model is already good in the domain we are interested in.\nIn our case, we want it to answer questions, while for other use cases, we might want it to follow instructions, in which case instruction tuning is a great idea.\nThe easiest way to achieve this is by continuing to train the language model with the language modeling objective on texts from the domain or task.\nThe [StackExchange dataset](https://huggingface.co/datasets/HuggingFaceH4/stack-exchange-preferences) is enormous (over 10 million instructions), so we can easily train the language model on a subset of it.\n\nThere is nothing special about fine-tuning the model before doing RLHF - it\u2019s just the causal language modeling objective from pretraining that we apply here.\nTo use the data efficiently, we use a technique called packing: instead of having one text per sample in the batch and then padding to either the longest text or the maximal context of the model, we concatenate a lot of texts with a EOS token in between and cut chunks of the context size to fill the batch without any padding.\n\n![chapter10_preprocessing-clm.png](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/blog/stackllama/chapter10_preprocessing-clm.png)\n\nWith this approach the training is much more efficient as each token that is passed through the model is also trained in contrast to padding tokens which are usually masked from the loss.\nIf you don't have much data and are more concerned about occasionally cutting off some tokens that are overflowing the context you can also use a classical data loader.\n\nThe packing is handled by the `ConstantLengthDataset` and we can then use the `Trainer` after loading the model with `peft`. First, we load the model in int8, prepare it for training, and then add the LoRA adapters."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 99ec0bee-4a71-4412-8a22-52c68e33f7ca
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/using_llama_models", "title": "Using LLaMA models with TRL", "tokens": 2435, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"99ec0bee-4a71-4412-8a22-52c68e33f7ca\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93a491eb-cddf-48d0-82d2-52bc28a0bdce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"259be3a8f639379b43879cb2f6ee34ff3313ac5605649be4013eb4b9921baa51\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"663c0a58-3176-4584-97e1-726d7fbb099a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"9509fc237237f3c3ade5ba77533858ca5f3b93aef3880ca159aaf2805cdab5c4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"926b4c30-c629-4484-a286-eae1c75e5491\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7f4f23ace1997b54897b81cdd5907dbd4d4f579deb2abf4ae745d5abe758e711\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3107, \"end_char_idx\": 6466, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "ref_doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "chroma:document": "In our case, we want it to answer questions, while for other use cases, we might want it to follow instructions, in which case instruction tuning is a great idea.\nThe easiest way to achieve this is by continuing to train the language model with the language modeling objective on texts from the domain or task.\nThe [StackExchange dataset](https://huggingface.co/datasets/HuggingFaceH4/stack-exchange-preferences) is enormous (over 10 million instructions), so we can easily train the language model on a subset of it.\n\nThere is nothing special about fine-tuning the model before doing RLHF - it\u2019s just the causal language modeling objective from pretraining that we apply here.\nTo use the data efficiently, we use a technique called packing: instead of having one text per sample in the batch and then padding to either the longest text or the maximal context of the model, we concatenate a lot of texts with a EOS token in between and cut chunks of the context size to fill the batch without any padding.\n\n![chapter10_preprocessing-clm.png](https://huggingface.co/datasets/trl-internal-testing/example-images/resolve/main/blog/stackllama/chapter10_preprocessing-clm.png)\n\nWith this approach the training is much more efficient as each token that is passed through the model is also trained in contrast to padding tokens which are usually masked from the loss.\nIf you don't have much data and are more concerned about occasionally cutting off some tokens that are overflowing the context you can also use a classical data loader.\n\nThe packing is handled by the `ConstantLengthDataset` and we can then use the `Trainer` after loading the model with `peft`. First, we load the model in int8, prepare it for training, and then add the LoRA adapters.\n\n```python\n# load model in 8bit\nmodel = AutoModelForCausalLM.from_pretrained(\n        args.model_path,\n        load_in_8bit=True,\n        device_map={\"\": Accelerator().local_process_index}\n    )\nmodel = prepare_model_for_kbit_training(model)\n\n# add LoRA to model\nlora_config = LoraConfig(\n    r=16,\n    lora_alpha=32,\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\n\nmodel = get_peft_model(model, config)\n```\n\nWe train the model for a few thousand steps with the causal language modeling objective and save the model.\nSince we will tune the model again with different objectives, we merge the adapter weights with the original model weights.\n\n**Disclaimer:** due to LLaMA's license, we release only the adapter weights for this and the model checkpoints in the following sections.\nYou can apply for access to the base model's weights by filling out Meta AI's [form](https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform) and then converting them to the \ud83e\udd17 Transformers format by running this [script](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py).\nNote that you'll also need to install \ud83e\udd17 Transformers from source until the `v4.28` is released.\n\nNow that we have fine-tuned the model for the task, we are ready to train a reward model.\n\n## Reward modeling and human preferences\n\nIn principle, we could fine-tune the model using RLHF directly with the human annotations.\nHowever, this would require us to send some samples to humans for rating after each optimization iteration."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 926b4c30-c629-4484-a286-eae1c75e5491
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/using_llama_models", "title": "Using LLaMA models with TRL", "tokens": 2435, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"926b4c30-c629-4484-a286-eae1c75e5491\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93a491eb-cddf-48d0-82d2-52bc28a0bdce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"259be3a8f639379b43879cb2f6ee34ff3313ac5605649be4013eb4b9921baa51\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"99ec0bee-4a71-4412-8a22-52c68e33f7ca\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"c7427b83232903f619956a99f62afed5cc562a42d6d1be472142251a6ef48346\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7d3c76ad-24a6-48ac-906b-aec5aa369d67\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bd9bf0586e0f93dcb99d47d2b41f50a94bdb98c1af488f69360ae58f3c50f37a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5393, \"end_char_idx\": 8718, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "ref_doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "chroma:document": "Since we will tune the model again with different objectives, we merge the adapter weights with the original model weights.\n\n**Disclaimer:** due to LLaMA's license, we release only the adapter weights for this and the model checkpoints in the following sections.\nYou can apply for access to the base model's weights by filling out Meta AI's [form](https://docs.google.com/forms/d/e/1FAIpQLSfqNECQnMkycAp2jP4Z9TFX0cGR4uf7b_fBxjY_OjhJILlKGA/viewform) and then converting them to the \ud83e\udd17 Transformers format by running this [script](https://github.com/huggingface/transformers/blob/main/src/transformers/models/llama/convert_llama_weights_to_hf.py).\nNote that you'll also need to install \ud83e\udd17 Transformers from source until the `v4.28` is released.\n\nNow that we have fine-tuned the model for the task, we are ready to train a reward model.\n\n## Reward modeling and human preferences\n\nIn principle, we could fine-tune the model using RLHF directly with the human annotations.\nHowever, this would require us to send some samples to humans for rating after each optimization iteration.\nThis is expensive and slow due to the number of training samples needed for convergence and the inherent latency of human reading and annotator speed.\n\nA trick that works well instead of direct feedback is training a reward model on human annotations collected before the RL loop.\nThe goal of the reward model is to imitate how a human would rate a text. There are several possible strategies to build a reward model: the most straightforward way would be to predict the annotation (e.g. a rating score or a binary value for \u201cgood\u201d/\u201dbad\u201d).\nIn practice, what works better is to predict the ranking of two examples, where the reward model is presented with two candidates `(y_k, y_j)` for a given prompt `x` and has to predict which one would be rated higher by a human annotator.\n\nWith the StackExchange dataset, we can infer which of the two answers was preferred by the users based on the score.\nWith that information and the loss defined above, we can then modify the `transformers.Trainer` by adding a custom loss function.\n\n```python\nclass RewardTrainer(Trainer):\n    def compute_loss(self, model, inputs, return_outputs=False):\n        rewards_j = model(input_ids=inputs[\"input_ids_j\"],  attention_mask=inputs[\"attention_mask_j\"])[0]\n        rewards_k = model(input_ids=inputs[\"input_ids_k\"], attention_mask=inputs[\"attention_mask_k\"])[0]\n        loss = -nn.functional.logsigmoid(rewards_j - rewards_k).mean()\n        if return_outputs:\n            return loss, {\"rewards_j\": rewards_j, \"rewards_k\": rewards_k}\n        return loss\n```\n\nWe utilize a subset of a 100,000 pair of candidates and evaluate on a held-out set of 50,000. With a modest training batch size of 4, we train the Llama model using the LoRA `peft` adapter for a single epoch using the Adam optimizer with BF16 precision. Our LoRA configuration is:\n\n```python\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_CLS,\n    inference_mode=False,\n    r=8,\n    lora_alpha=32,\n    lora_dropout=0.1,\n)\n```\nAs detailed in the next section, the resulting adapter can be merged into the frozen model and saved for further downstream use.\n\n## Reinforcement Learning from Human Feedback\n\nWith the fine-tuned language model and the reward model at hand, we are now ready to run the RL loop."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 7d3c76ad-24a6-48ac-906b-aec5aa369d67
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/using_llama_models", "title": "Using LLaMA models with TRL", "tokens": 2435, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"7d3c76ad-24a6-48ac-906b-aec5aa369d67\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"93a491eb-cddf-48d0-82d2-52bc28a0bdce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"259be3a8f639379b43879cb2f6ee34ff3313ac5605649be4013eb4b9921baa51\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"926b4c30-c629-4484-a286-eae1c75e5491\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/using_llama_models\", \"title\": \"Using LLaMA models with TRL\", \"tokens\": 2435, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b8b82bf0f5ebb3ff8c2585ac3349b3a1dccf60700b334c99e91001d612b776d5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7247, \"end_char_idx\": 10461, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "ref_doc_id": "93a491eb-cddf-48d0-82d2-52bc28a0bdce", "chroma:document": "With the StackExchange dataset, we can infer which of the two answers was preferred by the users based on the score.\nWith that information and the loss defined above, we can then modify the `transformers.Trainer` by adding a custom loss function.\n\n```python\nclass RewardTrainer(Trainer):\n    def compute_loss(self, model, inputs, return_outputs=False):\n        rewards_j = model(input_ids=inputs[\"input_ids_j\"],  attention_mask=inputs[\"attention_mask_j\"])[0]\n        rewards_k = model(input_ids=inputs[\"input_ids_k\"], attention_mask=inputs[\"attention_mask_k\"])[0]\n        loss = -nn.functional.logsigmoid(rewards_j - rewards_k).mean()\n        if return_outputs:\n            return loss, {\"rewards_j\": rewards_j, \"rewards_k\": rewards_k}\n        return loss\n```\n\nWe utilize a subset of a 100,000 pair of candidates and evaluate on a held-out set of 50,000. With a modest training batch size of 4, we train the Llama model using the LoRA `peft` adapter for a single epoch using the Adam optimizer with BF16 precision. Our LoRA configuration is:\n\n```python\npeft_config = LoraConfig(\n    task_type=TaskType.SEQ_CLS,\n    inference_mode=False,\n    r=8,\n    lora_alpha=32,\n    lora_dropout=0.1,\n)\n```\nAs detailed in the next section, the resulting adapter can be merged into the frozen model and saved for further downstream use.\n\n## Reinforcement Learning from Human Feedback\n\nWith the fine-tuned language model and the reward model at hand, we are now ready to run the RL loop. It follows roughly three steps:\n\n1. Generate responses from prompts,\n2. Rate the responses with the reward model,\n3. Run a reinforcement learning policy-optimization step with the ratings.\n\nThe Query and Response prompts are templated as follows before being tokenized and passed to the model:\n\n```bash\nQuestion: <Query>\n\nAnswer: <Response>\n```\n\nThe same template was used for SFT, RM and RLHF stages.\nOnce more, we utilize `peft` for memory-efficient training, which offers an extra advantage in the RLHF context.\nHere, the reference model and policy share the same base, the SFT model, which we load in 8-bit and freeze during training.\nWe exclusively optimize the policy's LoRA weights using PPO while sharing the base model's weights.\n\n```python\nfor epoch, batch in tqdm(enumerate(ppo_trainer.dataloader)):\n    question_tensors = batch[\"input_ids\"]\n\n\t# sample from the policy and to generate responses\n    response_tensors = ppo_trainer.generate(\n        question_tensors,\n        return_prompt=False,\n        length_sampler=output_length_sampler,\n        **generation_kwargs,\n    )\n    batch[\"response\"] = tokenizer.batch_decode(response_tensors, skip_special_tokens=True)\n\n    # Compute sentiment score\n    texts = [q + r for q, r in zip(batch[\"query\"], batch[\"response\"])]\n    pipe_outputs = sentiment_pipe(texts, **sent_kwargs)\n    rewards = [torch.tensor(output[0][\"score\"] - script_args.reward_baseline) for output in pipe_outputs]\n\n    # Run PPO step\n    stats = ppo_trainer.step(question_tensors, response_tensors, rewards)\n\t# Log stats to Wandb\n    ppo_trainer.log_stats(stats, batch, rewards)\n```\n\nFor the rest of the details and evaluation, please refer to our [blog post on StackLLaMA](https://huggingface.co/blog/stackllama)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 45215e04-47d0-40ef-ae22-74b441a4bf5a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/orpo_trainer", "title": "ORPO Trainer", "tokens": 990, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"45215e04-47d0-40ef-ae22-74b441a4bf5a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/orpo_trainer\", \"title\": \"ORPO Trainer\", \"tokens\": 990, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a69e62f2-903a-4445-a88f-27029ab80188\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/orpo_trainer\", \"title\": \"ORPO Trainer\", \"tokens\": 990, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0848c2f0388c0fdf544a01368ae241f8738c4e8a2d0a8be6c6d581963a0e5074\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c5aef82d-da6d-460d-a0a7-01095e38e68f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c5dbd118570e6d541077e2b9abccded09ab4f0618bafa3351c352b5dcb7e02bb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3296, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a69e62f2-903a-4445-a88f-27029ab80188", "doc_id": "a69e62f2-903a-4445-a88f-27029ab80188", "ref_doc_id": "a69e62f2-903a-4445-a88f-27029ab80188", "chroma:document": "# ORPO Trainer\n\n[Odds Ratio Preference Optimization](https://huggingface.co/papers/2403.07691) (ORPO) by Jiwoo Hong, Noah Lee, and James Thorne studies the crucial role of SFT within the context of preference alignment. Using preference data the method posits that a minor penalty for the disfavored generation together with a strong adaption signal to the chosen response via a simple log odds ratio term appended to the NLL loss is sufficient for preference-aligned SFT.\n\nThus ORPO is a reference model-free preference optimization algorithm eliminating the necessity for an additional preference alignment phase thus saving compute and memory.\n\nThe official code can be found [xfactlab/orpo](https://github.com/xfactlab/orpo).\n\n## Expected dataset format\n\nThe ORPO trainer expects a format identical to the DPO trainer, which should include three entries. These entries should be named as follows:\n\n- `prompt`\n- `chosen`\n- `rejected`\n\nfor example:\n\n```py\norpo_dataset_dict = {\n    \"prompt\": [\n        \"hello\",\n        \"how are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"chosen\": [\n        \"hi nice to meet you\",\n        \"I am fine\",\n        \"My name is Mary\",\n        \"My name is Mary\",\n        \"Python\",\n        \"Python\",\n        \"Java\",\n    ],\n    \"rejected\": [\n        \"leave me alone\",\n        \"I am not fine\",\n        \"Whats it to you?\",\n        \"I dont have a name\",\n        \"Javascript\",\n        \"C++\",\n        \"C++\",\n    ],\n}\n```\nwhere the `prompt` contains the context inputs, `chosen` contains the corresponding chosen responses and `rejected` contains the corresponding negative (rejected) responses. Note that a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n## Expected model format\nThe ORPO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `ORPOTrainer`\nFor a detailed example have a look at the `examples/scripts/orpo.py` script. At a high level we need to initialize the `ORPOTrainer` with a `model` we wish to train. **Note that ORPOTrainer eliminates the need to use the reference model, simplifying the optimization process.** The `beta` refers to the hyperparameter `lambda` in eq. (6) of the paper and refers to the weighting of the relative odd ratio loss in the standard cross-entropy loss used for SFT.\n\n```py\norpo_config = ORPOConfig(\n    beta=0.1, # the lambda/alpha hyperparameter in the paper/code\n)\n\norpo_trainer = ORPOTrainer(\n    model,\n    args=orpo_config,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\norpo_trainer.train()\n```\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.  \n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: c5aef82d-da6d-460d-a0a7-01095e38e68f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/orpo_trainer", "title": "ORPO Trainer", "tokens": 990, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"c5aef82d-da6d-460d-a0a7-01095e38e68f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/orpo_trainer\", \"title\": \"ORPO Trainer\", \"tokens\": 990, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a69e62f2-903a-4445-a88f-27029ab80188\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/orpo_trainer\", \"title\": \"ORPO Trainer\", \"tokens\": 990, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"0848c2f0388c0fdf544a01368ae241f8738c4e8a2d0a8be6c6d581963a0e5074\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"45215e04-47d0-40ef-ae22-74b441a4bf5a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/orpo_trainer\", \"title\": \"ORPO Trainer\", \"tokens\": 990, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"8bcf67deb1513035cdd218bdbfebe9fb007dcf41dcfba64b475f0c483c24fff5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1803, \"end_char_idx\": 4292, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a69e62f2-903a-4445-a88f-27029ab80188", "doc_id": "a69e62f2-903a-4445-a88f-27029ab80188", "ref_doc_id": "a69e62f2-903a-4445-a88f-27029ab80188", "chroma:document": "Note that a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n## Expected model format\nThe ORPO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `ORPOTrainer`\nFor a detailed example have a look at the `examples/scripts/orpo.py` script. At a high level we need to initialize the `ORPOTrainer` with a `model` we wish to train. **Note that ORPOTrainer eliminates the need to use the reference model, simplifying the optimization process.** The `beta` refers to the hyperparameter `lambda` in eq. (6) of the paper and refers to the weighting of the relative odd ratio loss in the standard cross-entropy loss used for SFT.\n\n```py\norpo_config = ORPOConfig(\n    beta=0.1, # the lambda/alpha hyperparameter in the paper/code\n)\n\norpo_trainer = ORPOTrainer(\n    model,\n    args=orpo_config,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\norpo_trainer.train()\n```\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.  \n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001).\n\n## Logging\n\nWhile training and evaluating we record the following reward metrics:\n\n* `rewards/chosen`: the mean log probabilities of the policy model for the chosen responses scaled by beta\n* `rewards/rejected`: the mean log probabilities of the policy model for the rejected responses scaled by beta\n* `rewards/accuracies`: mean of how often the chosen rewards are > than the corresponding rejected rewards\n* `rewards/margins`: the mean difference between the chosen and corresponding rejected rewards\n\n* `log_odds_chosen`: the mean log odds ratio of the chosen responses over the rejected responses\n\n* `log_odds_ratio`: the mean of the `log(sigmoid(log_odds_chosen))`\n\n* `nll_loss`: the mean negative log likelihood loss from the SFT part of the loss over chosen responses\n \n## ORPOTrainer\n\n[[autodoc]] ORPOTrainer\n\n\n## ORPOConfig\n\n[[autodoc]] ORPOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 82907709-23e9-4779-a60a-13ab92c8749f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/cpo_trainer", "title": "CPO Trainer", "tokens": 1413, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"82907709-23e9-4779-a60a-13ab92c8749f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"18aa29a8-b595-4c57-b0c7-b462e1ff90ca\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3c5ddf63cd64bb51aab8396cb67938996154764122c4eec95a7ea0c7144808b4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c470726f-f414-4d01-950c-7f7e3f3c0581\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5bdef4a7c1a944c265cc6e98acab01055a99a872e3d952acf76fb41de40c1d58\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3220, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "ref_doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "chroma:document": "# CPO Trainer\n\nContrastive Preference Optimization (CPO) as introduced in the paper [Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation](https://huggingface.co/papers/2401.08417) by Haoran Xu, Amr Sharaf, Yunmo Chen, Weiting Tan, Lingfeng Shen, Benjamin Van Durme, Kenton Murray, and Young Jin Kim. At a high-level, CPO  trains models to\navoid generating adequate, but not perfect translations in Machine Translation (MT) tasks. However, CPO is a general approximation to the DPO loss and can be applied to other domains like chat.\n\nCPO aims to mitigate two fundamental shortcomings of SFT. First, SFT\u2019s methodology of minimizing the discrepancy between predicted outputs and gold-standard references inherently caps model performance at the quality level of the training data. Secondly, SFT lacks a mechanism to prevent the model from rejecting mistakes in translations. The CPO objective is derived from the DPO objective.\n\n## SimPO\nThe [SimPO](https://huggingface.co/papers/2405.14734) method is also implemented in the `CPOTrainer`. SimPO is an alternative loss that adds a reward margin, allows for length normalization, and does not use BC regularization. To use this loss, we can use SimPO easily by turning on `loss_type=\"simpo\"` and `cpo_alpha=0` in the `CPOConfig`.\n\n## CPO-SimPO\nWe also offer the combined use of CPO and SimPO, which enables more stable training and improved performance. Learn more details at [CPO-SimPO Github](https://github.com/fe1ixxu/CPO_SIMPO). To use this method, simply enable SimPO by setting `loss_type=\"simpo\"` and a non-zero `cpo_alpha` in the CPOConfig.\n\n## Expected dataset format\n\nThe CPO trainer expects a format identical to the DPO trainer, which should include three entries. These entries should be named as follows:\n\n- `prompt`\n- `chosen`\n- `rejected`\n\nfor example:\n\n```py\ncpo_dataset_dict = {\n    \"prompt\": [\n        \"hello\",\n        \"how are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"chosen\": [\n        \"hi nice to meet you\",\n        \"I am fine\",\n        \"My name is Mary\",\n        \"My name is Mary\",\n        \"Python\",\n        \"Python\",\n        \"Java\",\n    ],\n    \"rejected\": [\n        \"leave me alone\",\n        \"I am not fine\",\n        \"Whats it to you?\",\n        \"I dont have a name\",\n        \"Javascript\",\n        \"C++\",\n        \"C++\",\n    ],\n}\n```\nwhere the `prompt` contains the context inputs, `chosen` contains the corresponding chosen responses and `rejected` contains the corresponding negative (rejected) responses. As can be seen a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n## Expected model format\nThe CPO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `CPOTrainer`\nFor a detailed example have a look at the `examples/scripts/cpo.py` script. At a high level we need to initialize the `CPOTrainer` with a `model` we wish to train."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: c470726f-f414-4d01-950c-7f7e3f3c0581
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/cpo_trainer", "title": "CPO Trainer", "tokens": 1413, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"c470726f-f414-4d01-950c-7f7e3f3c0581\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"18aa29a8-b595-4c57-b0c7-b462e1ff90ca\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3c5ddf63cd64bb51aab8396cb67938996154764122c4eec95a7ea0c7144808b4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"82907709-23e9-4779-a60a-13ab92c8749f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"583e153fe9a4edcb07f0e9d404e5129080c30f1f075d03666807a1f25a78fade\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"133146a5-95fb-43e2-9b9a-1c503f7ab994\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a4a643c321e03dbbea85d874c6790d571b1247ead0a31626af7f478c0a51519d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1648, \"end_char_idx\": 4748, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "ref_doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "chroma:document": "## Expected dataset format\n\nThe CPO trainer expects a format identical to the DPO trainer, which should include three entries. These entries should be named as follows:\n\n- `prompt`\n- `chosen`\n- `rejected`\n\nfor example:\n\n```py\ncpo_dataset_dict = {\n    \"prompt\": [\n        \"hello\",\n        \"how are you\",\n        \"What is your name?\",\n        \"What is your name?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n        \"Which is the best programming language?\",\n    ],\n    \"chosen\": [\n        \"hi nice to meet you\",\n        \"I am fine\",\n        \"My name is Mary\",\n        \"My name is Mary\",\n        \"Python\",\n        \"Python\",\n        \"Java\",\n    ],\n    \"rejected\": [\n        \"leave me alone\",\n        \"I am not fine\",\n        \"Whats it to you?\",\n        \"I dont have a name\",\n        \"Javascript\",\n        \"C++\",\n        \"C++\",\n    ],\n}\n```\nwhere the `prompt` contains the context inputs, `chosen` contains the corresponding chosen responses and `rejected` contains the corresponding negative (rejected) responses. As can be seen a prompt can have multiple responses and this is reflected in the entries being repeated in the dictionary's value arrays.\n\n## Expected model format\nThe CPO trainer expects a model of `AutoModelForCausalLM`, compared to PPO that expects `AutoModelForCausalLMWithValueHead` for the value function.\n\n## Using the `CPOTrainer`\nFor a detailed example have a look at the `examples/scripts/cpo.py` script. At a high level we need to initialize the `CPOTrainer` with a `model` we wish to train. **Note that CPOTrainer eliminates the need to use the reference model, simplifying the optimization process.** The `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above.\n\n```py\ncpo_config = CPOConfig(\n    beta=0.1,\n)\n\ncpo_trainer = CPOTrainer(\n    model,\n    args=cpo_config,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\ncpo_trainer.train()\n```\n\n## Loss functions\n\nGiven the preference data, the `CPOTrainer` uses the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression.\n\nThe [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. The `CPOTrainer` can be switched to this loss via the `loss_type=\"hinge\"` argument and the `beta` in this case is the reciprocal of the margin.\n\nThe [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the CPO algorithms and identify an issue with overfitting and propose an alternative loss which can be used via the `loss_type=\"ipo\"` argument to the trainer. Note that the `beta`  parameter is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair and thus the smaller the `beta` the larger this gaps is. As per the paper the loss is averaged over log-likelihoods of the completion (unlike CPO which is summed only)."}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 133146a5-95fb-43e2-9b9a-1c503f7ab994
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/cpo_trainer", "title": "CPO Trainer", "tokens": 1413, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"133146a5-95fb-43e2-9b9a-1c503f7ab994\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"18aa29a8-b595-4c57-b0c7-b462e1ff90ca\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"3c5ddf63cd64bb51aab8396cb67938996154764122c4eec95a7ea0c7144808b4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c470726f-f414-4d01-950c-7f7e3f3c0581\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/cpo_trainer\", \"title\": \"CPO Trainer\", \"tokens\": 1413, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"5b6e93d26174dc3de4af13eac33f897bbc1b1f9422d8c5c0c64f0e10f5cd0712\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3221, \"end_char_idx\": 5973, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "ref_doc_id": "18aa29a8-b595-4c57-b0c7-b462e1ff90ca", "chroma:document": "**Note that CPOTrainer eliminates the need to use the reference model, simplifying the optimization process.** The `beta` refers to the hyperparameter of the implicit reward, and the dataset contains the 3 entries listed above.\n\n```py\ncpo_config = CPOConfig(\n    beta=0.1,\n)\n\ncpo_trainer = CPOTrainer(\n    model,\n    args=cpo_config,\n    train_dataset=train_dataset,\n    tokenizer=tokenizer,\n)\n```\nAfter this one can then call:\n\n```py\ncpo_trainer.train()\n```\n\n## Loss functions\n\nGiven the preference data, the `CPOTrainer` uses the sigmoid loss on the normalized likelihood via the `logsigmoid` to fit a logistic regression.\n\nThe [RSO](https://huggingface.co/papers/2309.06657) authors propose to use a hinge loss on the normalized likelihood from the [SLiC](https://huggingface.co/papers/2305.10425) paper. The `CPOTrainer` can be switched to this loss via the `loss_type=\"hinge\"` argument and the `beta` in this case is the reciprocal of the margin.\n\nThe [IPO](https://huggingface.co/papers/2310.12036) authors provide a deeper theoretical understanding of the CPO algorithms and identify an issue with overfitting and propose an alternative loss which can be used via the `loss_type=\"ipo\"` argument to the trainer. Note that the `beta`  parameter is the reciprocal of the gap between the log-likelihood ratios of the chosen vs the rejected completion pair and thus the smaller the `beta` the larger this gaps is. As per the paper the loss is averaged over log-likelihoods of the completion (unlike CPO which is summed only).\n\n### For Mixture of Experts Models: Enabling the auxiliary loss\n\nMOEs are the most efficient if the load is about equally distributed between experts.  \nTo ensure that we train MOEs similarly during preference-tuning, it is beneficial to add the auxiliary loss from the load balancer to the final loss.  \n\nThis option is enabled by setting `output_router_logits=True` in the model config (e.g. MixtralConfig).  \nTo scale how much the auxiliary loss contributes to the total loss, use the hyperparameter `router_aux_loss_coef=...` (default: 0.001).\n\n## Logging\n\nWhile training and evaluating we record the following reward metrics:\n\n* `rewards/chosen`: the mean log probabilities of the policy model for the chosen responses scaled by beta\n* `rewards/rejected`: the mean log probabilities of the policy model for the rejected responses scaled by beta\n* `rewards/accuracies`: mean of how often the chosen rewards are > than the corresponding rejected rewards\n* `rewards/margins`: the mean difference between the chosen and corresponding rejected rewards\n* `nll_loss`: the mean negative log likelihood loss of the policy model for the chosen responses\n\n## CPOTrainer\n\n[[autodoc]] CPOTrainer\n\n## CPOConfig\n\n[[autodoc]] CPOConfig"}


created_at: 2024-08-26 20:30:10
topic: persistent://default/default/b0d61235-9c1a-4b51-b8f3-dd66e1fe516e
id: 1dc69009-12f9-4ee4-a6d5-f096f370a2cd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/trl/installation", "title": "Installation", "tokens": 118, "retrieve_doc": true, "source": "trl", "_node_content": "{\"id_\": \"1dc69009-12f9-4ee4-a6d5-f096f370a2cd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/installation\", \"title\": \"Installation\", \"tokens\": 118, \"retrieve_doc\": true, \"source\": \"trl\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"76223a48-8c3a-479d-8f7d-e10365954ad9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/trl/installation\", \"title\": \"Installation\", \"tokens\": 118, \"retrieve_doc\": true, \"source\": \"trl\"}, \"hash\": \"b9e9be9347caebecb81b0e038597148804af5127cf1f4791b1517de2d7f8d736\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 476, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "76223a48-8c3a-479d-8f7d-e10365954ad9", "doc_id": "76223a48-8c3a-479d-8f7d-e10365954ad9", "ref_doc_id": "76223a48-8c3a-479d-8f7d-e10365954ad9", "chroma:document": "# Installation\nYou can install TRL either from pypi or from source:\n\n## pypi\nInstall the library with pip:\n\n```bash\npip install trl\n```\n\n### Source\nYou can also install the latest version from source. First clone the repo and then run the installation with `pip`:\n\n```bash\ngit clone https://github.com/huggingface/trl.git\ncd trl/\npip install -e .\n```\n\nIf you want the development install you can replace the pip install with the following:\n\n```bash\npip install -e \".[dev]\"\n```"}


