# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e60964f8-d804-4e0b-a73a-7b31a7c8a36f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/quicktour", "title": "Quicktour", "tokens": 1893, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e60964f8-d804-4e0b-a73a-7b31a7c8a36f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fd743c14-c813-47ec-800f-c8f165074de5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d0903411981d954f5a738ac0d7c3c18955243168c7170ebdd7501154bbb76131\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e2771e1b-5d3a-4d8d-857c-071511691fc0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"52465a1ba212d545de70a78329bcec45d4e223d1556da7c2f0e4dad53f512370\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2873, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fd743c14-c813-47ec-800f-c8f165074de5", "doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "ref_doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "chroma:document": "# Quicktour\n\nPEFT offers parameter-efficient methods for finetuning large pretrained models. The traditional paradigm is to finetune all of a model's parameters for each downstream task, but this is becoming exceedingly costly and impractical because of the enormous number of parameters in models today. Instead, it is more efficient to train a smaller number of prompt parameters or use a reparametrization method like low-rank adaptation (LoRA) to reduce the number of trainable parameters.\n\nThis quicktour will show you PEFT's main features and how you can train or run inference on large models that would typically be inaccessible on consumer devices.\n\n## Train\n\nEach PEFT method is defined by a [`PeftConfig`] class that stores all the important parameters for building a [`PeftModel`]. For example, to train with LoRA, load and create a [`LoraConfig`] class and specify the following parameters:\n\n- `task_type`: the task to train for (sequence-to-sequence language modeling in this case)\n- `inference_mode`: whether you're using the model for inference or not\n- `r`: the dimension of the low-rank matrices\n- `lora_alpha`: the scaling factor for the low-rank matrices\n- `lora_dropout`: the dropout probability of the LoRA layers\n\n```python\nfrom peft import LoraConfig, TaskType\n\npeft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1)\n```\n\n<Tip>\n\nSee the [`LoraConfig`] reference for more details about other parameters you can adjust, such as the modules to target or the bias type.\n\n</Tip>\n\nOnce the [`LoraConfig`] is setup, create a [`PeftModel`] with the [`get_peft_model`] function. It takes a base model - which you can load from the Transformers library - and the [`LoraConfig`] containing the parameters for how to configure a model for training with LoRA.\n\nLoad the base model you want to finetune.\n\n```python\nfrom transformers import AutoModelForSeq2SeqLM\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"bigscience/mt0-large\")\n```\n\nWrap the base model and `peft_config` with the [`get_peft_model`] function to create a [`PeftModel`]. To get a sense of the number of trainable parameters in your model, use the [`print_trainable_parameters`] method.\n\n```python\nfrom peft import get_peft_model\n\nmodel = get_peft_model(model, peft_config)\nmodel.print_trainable_parameters()\n\"output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282\"\n```\n\nOut of [bigscience/mt0-large's](https://huggingface.co/bigscience/mt0-large) 1.2B parameters, you're only training 0.19% of them!\n\nThat is it \ud83c\udf89! Now you can train the model with the Transformers [`~transformers.Trainer`], Accelerate, or any custom PyTorch training loop.\n\nFor example, to train with the [`~transformers.Trainer`] class, setup a [`~transformers.TrainingArguments`] class with some training hyperparameters."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e2771e1b-5d3a-4d8d-857c-071511691fc0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/quicktour", "title": "Quicktour", "tokens": 1893, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e2771e1b-5d3a-4d8d-857c-071511691fc0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fd743c14-c813-47ec-800f-c8f165074de5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d0903411981d954f5a738ac0d7c3c18955243168c7170ebdd7501154bbb76131\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e60964f8-d804-4e0b-a73a-7b31a7c8a36f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"6929cf04ae57218447b5dd4f0ea8ac458ab74880d41eef190c5ff1fcdb5023fc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1690459e-b537-4d8e-ba73-a97ad46ce3d0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f3e629583225ef8ce5afc48558446a515bfc2495a614ca6951c5ea0c0181fe45\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1554, \"end_char_idx\": 4554, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fd743c14-c813-47ec-800f-c8f165074de5", "doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "ref_doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "chroma:document": "</Tip>\n\nOnce the [`LoraConfig`] is setup, create a [`PeftModel`] with the [`get_peft_model`] function. It takes a base model - which you can load from the Transformers library - and the [`LoraConfig`] containing the parameters for how to configure a model for training with LoRA.\n\nLoad the base model you want to finetune.\n\n```python\nfrom transformers import AutoModelForSeq2SeqLM\n\nmodel = AutoModelForSeq2SeqLM.from_pretrained(\"bigscience/mt0-large\")\n```\n\nWrap the base model and `peft_config` with the [`get_peft_model`] function to create a [`PeftModel`]. To get a sense of the number of trainable parameters in your model, use the [`print_trainable_parameters`] method.\n\n```python\nfrom peft import get_peft_model\n\nmodel = get_peft_model(model, peft_config)\nmodel.print_trainable_parameters()\n\"output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282\"\n```\n\nOut of [bigscience/mt0-large's](https://huggingface.co/bigscience/mt0-large) 1.2B parameters, you're only training 0.19% of them!\n\nThat is it \ud83c\udf89! Now you can train the model with the Transformers [`~transformers.Trainer`], Accelerate, or any custom PyTorch training loop.\n\nFor example, to train with the [`~transformers.Trainer`] class, setup a [`~transformers.TrainingArguments`] class with some training hyperparameters.\n\n```py\ntraining_args = TrainingArguments(\n    output_dir=\"your-name/bigscience/mt0-large-lora\",\n    learning_rate=1e-3,\n    per_device_train_batch_size=32,\n    per_device_eval_batch_size=32,\n    num_train_epochs=2,\n    weight_decay=0.01,\n    evaluation_strategy=\"epoch\",\n    save_strategy=\"epoch\",\n    load_best_model_at_end=True,\n)\n```\n\nPass the model, training arguments, dataset, tokenizer, and any other necessary component to the [`~transformers.Trainer`], and call [`~transformers.Trainer.train`] to start training.\n\n```py\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics,\n)\n\ntrainer.train()\n```\n\n### Save model\n\nAfter your model is finished training, you can save your model to a directory using the [`~transformers.PreTrainedModel.save_pretrained`] function.\n\n```py\nmodel.save_pretrained(\"output_dir\")\n```\n\nYou can also save your model to the Hub (make sure you're logged in to your Hugging Face account first) with the [`~transformers.PreTrainedModel.push_to_hub`] function.\n\n```python\nfrom huggingface_hub import notebook_login\n\nnotebook_login()\nmodel.push_to_hub(\"your-name/bigscience/mt0-large-lora\")\n```\n\nBoth methods only save the extra PEFT weights that were trained, meaning it is super efficient to store, transfer, and load. For example, this [facebook/opt-350m](https://huggingface.co/ybelkada/opt-350m-lora) model trained with LoRA only contains two files: `adapter_config.json` and `adapter_model.safetensors`. The `adapter_model.safetensors` file is just 6.3MB!"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 1690459e-b537-4d8e-ba73-a97ad46ce3d0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/quicktour", "title": "Quicktour", "tokens": 1893, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"1690459e-b537-4d8e-ba73-a97ad46ce3d0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fd743c14-c813-47ec-800f-c8f165074de5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d0903411981d954f5a738ac0d7c3c18955243168c7170ebdd7501154bbb76131\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e2771e1b-5d3a-4d8d-857c-071511691fc0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a9cb96c38fcef8cc734c65239bfde6284c0b9d2c7a1cbbd364f113bd0fbf878e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6f8f4b0b-d9fb-42fd-bb79-697b3390e5f0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"572d7321cc13ce3ef0bab8c3c29bf6d84a7ca695893a1ca506cafcef437ad491\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3397, \"end_char_idx\": 6526, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fd743c14-c813-47ec-800f-c8f165074de5", "doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "ref_doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "chroma:document": "```py\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    train_dataset=tokenized_datasets[\"train\"],\n    eval_dataset=tokenized_datasets[\"test\"],\n    tokenizer=tokenizer,\n    data_collator=data_collator,\n    compute_metrics=compute_metrics,\n)\n\ntrainer.train()\n```\n\n### Save model\n\nAfter your model is finished training, you can save your model to a directory using the [`~transformers.PreTrainedModel.save_pretrained`] function.\n\n```py\nmodel.save_pretrained(\"output_dir\")\n```\n\nYou can also save your model to the Hub (make sure you're logged in to your Hugging Face account first) with the [`~transformers.PreTrainedModel.push_to_hub`] function.\n\n```python\nfrom huggingface_hub import notebook_login\n\nnotebook_login()\nmodel.push_to_hub(\"your-name/bigscience/mt0-large-lora\")\n```\n\nBoth methods only save the extra PEFT weights that were trained, meaning it is super efficient to store, transfer, and load. For example, this [facebook/opt-350m](https://huggingface.co/ybelkada/opt-350m-lora) model trained with LoRA only contains two files: `adapter_config.json` and `adapter_model.safetensors`. The `adapter_model.safetensors` file is just 6.3MB!\n\n<div class=\"flex flex-col justify-center\">\n  <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/PEFT-hub-screenshot.png\"/>\n  <figcaption class=\"text-center\">The adapter weights for a opt-350m model stored on the Hub are only ~6MB compared to the full size of the model weights, which can be ~700MB.</figcaption>\n</div>\n\n## Inference\n\n<Tip>\n\nTake a look at the [AutoPeftModel](package_reference/auto_class) API reference for a complete list of available `AutoPeftModel` classes.\n\n</Tip>\n\nEasily load any PEFT-trained model for inference with the [`AutoPeftModel`] class and the [`~transformers.PreTrainedModel.from_pretrained`] method:\n\n```py\nfrom peft import AutoPeftModelForCausalLM\nfrom transformers import AutoTokenizer\nimport torch\n\nmodel = AutoPeftModelForCausalLM.from_pretrained(\"ybelkada/opt-350m-lora\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\nmodel = model.to(\"cuda\")\nmodel.eval()\ninputs = tokenizer(\"Preheat the oven to 350 degrees and place the cookie dough\", return_tensors=\"pt\")\n\noutputs = model.generate(input_ids=inputs[\"input_ids\"].to(\"cuda\"), max_new_tokens=50)\nprint(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0])\n\n\"Preheat the oven to 350 degrees and place the cookie dough in the center of the oven. In a large bowl, combine the flour, baking powder, baking soda, salt, and cinnamon. In a separate bowl, combine the egg yolks, sugar, and vanilla.\"\n```\n\nFor other tasks that aren't explicitly supported with an `AutoPeftModelFor` class - such as automatic speech recognition - you can still use the base [`AutoPeftModel`] class to load a model for the task.\n\n```py\nfrom peft import AutoPeftModel\n\nmodel = AutoPeftModel.from_pretrained(\"smangrul/openai-whisper-large-v2-LORA-colab\")\n```\n\n## Next steps\n\nNow that you've seen how to train a model with one of the PEFT methods, we encourage you to try out some of the other methods like prompt tuning."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 6f8f4b0b-d9fb-42fd-bb79-697b3390e5f0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/quicktour", "title": "Quicktour", "tokens": 1893, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"6f8f4b0b-d9fb-42fd-bb79-697b3390e5f0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fd743c14-c813-47ec-800f-c8f165074de5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d0903411981d954f5a738ac0d7c3c18955243168c7170ebdd7501154bbb76131\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1690459e-b537-4d8e-ba73-a97ad46ce3d0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/quicktour\", \"title\": \"Quicktour\", \"tokens\": 1893, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"75c40fd9e2011fae6717e67539ad766dc18e4c28aa802b99560d5fa4b142eb58\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5077, \"end_char_idx\": 7109, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fd743c14-c813-47ec-800f-c8f165074de5", "doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "ref_doc_id": "fd743c14-c813-47ec-800f-c8f165074de5", "chroma:document": "</Tip>\n\nEasily load any PEFT-trained model for inference with the [`AutoPeftModel`] class and the [`~transformers.PreTrainedModel.from_pretrained`] method:\n\n```py\nfrom peft import AutoPeftModelForCausalLM\nfrom transformers import AutoTokenizer\nimport torch\n\nmodel = AutoPeftModelForCausalLM.from_pretrained(\"ybelkada/opt-350m-lora\")\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\n\nmodel = model.to(\"cuda\")\nmodel.eval()\ninputs = tokenizer(\"Preheat the oven to 350 degrees and place the cookie dough\", return_tensors=\"pt\")\n\noutputs = model.generate(input_ids=inputs[\"input_ids\"].to(\"cuda\"), max_new_tokens=50)\nprint(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0])\n\n\"Preheat the oven to 350 degrees and place the cookie dough in the center of the oven. In a large bowl, combine the flour, baking powder, baking soda, salt, and cinnamon. In a separate bowl, combine the egg yolks, sugar, and vanilla.\"\n```\n\nFor other tasks that aren't explicitly supported with an `AutoPeftModelFor` class - such as automatic speech recognition - you can still use the base [`AutoPeftModel`] class to load a model for the task.\n\n```py\nfrom peft import AutoPeftModel\n\nmodel = AutoPeftModel.from_pretrained(\"smangrul/openai-whisper-large-v2-LORA-colab\")\n```\n\n## Next steps\n\nNow that you've seen how to train a model with one of the PEFT methods, we encourage you to try out some of the other methods like prompt tuning. The steps are very similar to the ones shown in the quicktour:\n\n1. prepare a [`PeftConfig`] for a PEFT method\n2. use the [`get_peft_model`] method to create a [`PeftModel`] from the configuration and base model\n\nThen you can train it however you like! To load a PEFT model for inference, you can use the [`AutoPeftModel`] class.\n\nFeel free to also take a look at the task guides if you're interested in training a model with another PEFT method for a specific task such as semantic segmentation, multilingual automatic speech recognition, DreamBooth, token classification, and more."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 55f65d68-4c00-4311-aa9a-df4020c3079b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/install", "title": "Installation", "tokens": 376, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"55f65d68-4c00-4311-aa9a-df4020c3079b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/install\", \"title\": \"Installation\", \"tokens\": 376, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"07bfce4b-2473-4bd6-8c49-dabc113e235f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/install\", \"title\": \"Installation\", \"tokens\": 376, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"e572d410e2768b9554d546daf644a4667f654d04a41e796cb484d6bc3de05ad4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 822, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "07bfce4b-2473-4bd6-8c49-dabc113e235f", "doc_id": "07bfce4b-2473-4bd6-8c49-dabc113e235f", "ref_doc_id": "07bfce4b-2473-4bd6-8c49-dabc113e235f", "chroma:document": "# Installation\n\nBefore you start, you will need to setup your environment, install the appropriate packages, and configure \ud83e\udd17 PEFT. \ud83e\udd17 PEFT is tested on **Python 3.8+**.\n\n\ud83e\udd17 PEFT is available on PyPI, as well as GitHub:\n\n## PyPI\n\nTo install \ud83e\udd17 PEFT from PyPI:\n\n```bash\npip install peft\n```\n\n## Source\n\nNew features that haven't been released yet are added every day, which also means there may be some bugs. To try them out, install from the GitHub repository:\n\n```bash\npip install git+https://github.com/huggingface/peft\n```\n\nIf you're working on contributing to the library or wish to play with the source code and see live \nresults as you run the code, an editable version can be installed from a locally-cloned version of the \nrepository:\n\n```bash\ngit clone https://github.com/huggingface/peft\ncd peft\npip install -e .\n```"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: f656ff25-b135-4c2e-9a71-bb94cfe3d39c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/index", "title": "PEFT", "tokens": 907, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"f656ff25-b135-4c2e-9a71-bb94cfe3d39c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/index\", \"title\": \"PEFT\", \"tokens\": 907, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f42f6ce0-9d4a-4895-9084-84c2374e06b5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/index\", \"title\": \"PEFT\", \"tokens\": 907, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"15571d4d15958305baaf78e90f114a6be4086568c6dff4f01acd15b535e55179\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2971, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f42f6ce0-9d4a-4895-9084-84c2374e06b5", "doc_id": "f42f6ce0-9d4a-4895-9084-84c2374e06b5", "ref_doc_id": "f42f6ce0-9d4a-4895-9084-84c2374e06b5", "chroma:document": "# PEFT\n\n\ud83e\udd17 PEFT (Parameter-Efficient Fine-Tuning) is a library for efficiently adapting large pretrained models to various downstream applications without fine-tuning all of a model's parameters because it is prohibitively costly. PEFT methods only fine-tune a small number of (extra) model parameters - significantly decreasing computational and storage costs - while yielding performance comparable to a fully fine-tuned model. This makes it more accessible to train and store large language models (LLMs) on consumer hardware.\n\nPEFT is integrated with the Transformers, Diffusers, and Accelerate libraries to provide a faster and easier way to load, train, and use large models for inference.\n\n<div class=\"mt-10\">\n  <div class=\"w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-2 md:gap-y-4 md:gap-x-5\">\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"quicktour\"\n      ><div class=\"w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed\">Get started</div>\n      <p class=\"text-gray-700\">Start here if you're new to \ud83e\udd17 PEFT to get an overview of the library's main features, and how to train a model with a PEFT method.</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"./task_guides/image_classification_lora\"\n      ><div class=\"w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed\">How-to guides</div>\n      <p class=\"text-gray-700\">Practical guides demonstrating how to apply various PEFT methods across different types of tasks like image classification, causal language modeling, automatic speech recognition, and more. Learn how to use \ud83e\udd17 PEFT with the DeepSpeed and Fully Sharded Data Parallel scripts.</p>\n    </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"./conceptual_guides/lora\"\n      ><div class=\"w-full text-center bg-gradient-to-br from-pink-400 to-pink-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed\">Conceptual guides</div>\n      <p class=\"text-gray-700\">Get a better theoretical understanding of how LoRA and various soft prompting methods help reduce the number of trainable parameters to make training more efficient.</p>\n   </a>\n    <a class=\"!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg\" href=\"./package_reference/config\"\n      ><div class=\"w-full text-center bg-gradient-to-br from-purple-400 to-purple-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed\">Reference</div>\n      <p class=\"text-gray-700\">Technical descriptions of how \ud83e\udd17 PEFT classes and methods work.</p>\n    </a>\n  </div>\n</div>\n\n<iframe\n\tsrc=\"https://stevhliu-peft-methods.hf.space\"\n\tframeborder=\"0\"\n\twidth=\"850\"\n\theight=\"620\"\n></iframe>"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: c5fd1713-f7c1-4eb6-bde3-972691b97e3b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/ia3", "title": "IA3", "tokens": 903, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"c5fd1713-f7c1-4eb6-bde3-972691b97e3b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/ia3\", \"title\": \"IA3\", \"tokens\": 903, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b80faa0b-c7f1-4947-959a-e18e7a0933b4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/ia3\", \"title\": \"IA3\", \"tokens\": 903, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3367fb90ef387daaa9d3d21b5b1a813e22e5ac363b219faf6ab1dae58ad01ab3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3375, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b80faa0b-c7f1-4947-959a-e18e7a0933b4", "doc_id": "b80faa0b-c7f1-4947-959a-e18e7a0933b4", "ref_doc_id": "b80faa0b-c7f1-4947-959a-e18e7a0933b4", "chroma:document": "# IA3 \n\nThis conceptual guide gives a brief overview of [IA3](https://arxiv.org/abs/2205.05638), a parameter-efficient fine tuning technique that is \nintended to improve over [LoRA](./lora).\n\nTo make fine-tuning more efficient, IA3 (Infused Adapter by Inhibiting and Amplifying Inner Activations) \nrescales inner activations with learned vectors. These learned vectors are injected in the attention and feedforward modules \nin a typical transformer-based architecture. These learned vectors are the only trainable parameters during fine-tuning, and thus the original \nweights remain frozen. Dealing with learned vectors (as opposed to learned low-rank updates to a weight matrix like LoRA)\nkeeps the number of trainable parameters much smaller. \n\nBeing similar to LoRA, IA3 carries many of the same advantages: \n\n* IA3 makes fine-tuning more efficient by drastically reducing the number of trainable parameters. (For T0, an IA3 model only has about 0.01% trainable parameters, while even LoRA has > 0.1%)\n* The original pre-trained weights are kept frozen, which means you can have multiple lightweight and portable IA3 models for various downstream tasks built on top of them.\n* Performance of models fine-tuned using IA3 is comparable to the performance of fully fine-tuned models.\n* IA3 does not add any inference latency because adapter weights can be merged with the base model.\n\nIn principle, IA3 can be applied to any subset of weight matrices in a neural network to reduce the number of trainable\nparameters. Following the authors' implementation, IA3 weights are added to the key, value and feedforward layers\nof a Transformer model. To be specific, for transformer models, IA3 weights are added to the outputs of key and value layers, and to the input of the second feedforward layer\nin each transformer block.\n\nGiven the target layers for injecting IA3 parameters, the number of trainable parameters\ncan be determined based on the size of the weight matrices.\n\n\n## Common IA3 parameters in PEFT\n\nAs with other methods supported by PEFT, to fine-tune a model using IA3, you need to:\n\n1. Instantiate a base model.\n2. Create a configuration (`IA3Config`) where you define IA3-specific parameters.\n3. Wrap the base model with `get_peft_model()` to get a trainable `PeftModel`.\n4. Train the `PeftModel` as you normally would train the base model.\n\n`IA3Config` allows you to control how IA3 is applied to the base model through the following parameters:\n\n- `target_modules`: The modules (for example, attention blocks) to apply the IA3 vectors.\n- `feedforward_modules`: The list of modules to be treated as feedforward layers in `target_modules`. While learned vectors are multiplied with\nthe output activation for attention blocks, the vectors are multiplied with the input for classic feedforward layers. Note that `feedforward_modules` must be a subset of `target_modules`.\n- `modules_to_save`: List of modules apart from IA3 layers to be set as trainable and saved in the final checkpoint. These typically include model's custom head that is randomly initialized for the fine-tuning task.\n\n## Example Usage\n\nFor the task of sequence classification, one can initialize the IA3 config for a Llama model as follows:\n\n```py\npeft_config = IA3Config(\n    task_type=TaskType.SEQ_CLS, target_modules=[\"k_proj\", \"v_proj\", \"down_proj\"], feedforward_modules=[\"down_proj\"]\n)\n```"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 221a9d58-845b-43ed-b85a-e3273632e65c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/oft", "title": "Orthogonal Finetuning (OFT and BOFT)", "tokens": 1680, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"221a9d58-845b-43ed-b85a-e3273632e65c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/oft\", \"title\": \"Orthogonal Finetuning (OFT and BOFT)\", \"tokens\": 1680, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a879ee8e-9618-416c-8cae-c2820b9c1443\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/oft\", \"title\": \"Orthogonal Finetuning (OFT and BOFT)\", \"tokens\": 1680, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"422d6931e7d2f25de7bdf990eedef1ceb167fe303adaf96752e07e0437556830\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"10a15733-fef3-44c8-b074-bf6e0a6e0657\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5675374c1cdb06f2c59bab06616532dc5d719a97851579582605be3407d94740\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3547, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "doc_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "ref_doc_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "chroma:document": "# Orthogonal Finetuning (OFT and BOFT) \n\nThis conceptual guide gives a brief overview of [OFT](https://arxiv.org/abs/2306.07280) and [BOFT](https://arxiv.org/abs/2311.06243), a parameter-efficient fine-tuning technique that utilizes orthogonal matrix to multiplicatively transform the pretrained weight matrices.\n\nTo achieve efficient fine-tuning, OFT represents the weight updates with an orthogonal transformation. The orthogonal transformation is parameterized by an orthogonal matrix multiplied to the pretrained weight matrix. These new matrices can be trained to adapt to the new data while keeping the overall number of changes low. The original weight matrix remains frozen and doesn\u2019t receive any further adjustments. To produce the final results, both the original and the adapted weights are multiplied togethor.\n\nOrthogonal Butterfly (BOFT) generalizes OFT with Butterfly factorization and further improves its parameter efficiency and finetuning flexibility. In short, OFT can be viewed as a special case of BOFT. Different from LoRA that uses additive low-rank weight updates, BOFT uses multiplicative orthogonal weight updates. The comparison is shown below.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://raw.githubusercontent.com/wy1iu/butterfly-oft/main/assets/BOFT_comparison.png\"/>\n</div>\n\n\nBOFT has some advantages compared to LoRA: \n\n* BOFT proposes a simple yet generic way to finetune pretrained models to downstream tasks, yielding a better preservation of pretraining knowledge and a better parameter efficiency.\n* Through the orthogonality, BOFT introduces a structural constraint, i.e., keeping the [hyperspherical energy](https://arxiv.org/abs/1805.09298) unchanged during finetuning. This can effectively reduce the forgetting of pretraining knowledge.\n* BOFT uses the butterfly factorization to efficiently parameterize the orthogonal matrix, which yields a compact yet expressive learning space (i.e., hypothesis class).\n* The sparse matrix decomposition in BOFT brings in additional inductive biases that are beneficial to generalization.\n\nIn principle, BOFT can be applied to any subset of weight matrices in a neural network to reduce the number of trainable parameters. Given the target layers for injecting BOFT parameters, the number of trainable parameters can be determined based on the size of the weight matrices.\n\n## Merge OFT/BOFT weights into the base model\n\nSimilar to LoRA, the weights learned by OFT/BOFT can be integrated into the pretrained weight matrices using the merge_and_unload() function. This function merges the adapter weights with the base model which allows you to effectively use the newly merged model as a standalone model.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://raw.githubusercontent.com/wy1iu/butterfly-oft/main/assets/boft_merge.png\"/>\n</div>\n\nThis works because during training, the orthogonal weight matrix (R in the diagram above) and the pretrained weight matrices are separate. But once training is complete, these weights can actually be merged (multiplied) into a new weight matrix that is equivalent.\n\n## Utils for OFT / BOFT\n\n### Common OFT / BOFT parameters in PEFT\n\nAs with other methods supported by PEFT, to fine-tune a model using OFT or BOFT, you need to:\n\n1. Instantiate a base model.\n2. Create a configuration (`OFTConfig` or `BOFTConfig`) where you define OFT/BOFT-specific parameters.\n3. Wrap the base model with `get_peft_model()` to get a trainable `PeftModel`.\n4. Train the `PeftModel` as you normally would train the base model."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 10a15733-fef3-44c8-b074-bf6e0a6e0657
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/oft", "title": "Orthogonal Finetuning (OFT and BOFT)", "tokens": 1680, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"10a15733-fef3-44c8-b074-bf6e0a6e0657\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/oft\", \"title\": \"Orthogonal Finetuning (OFT and BOFT)\", \"tokens\": 1680, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a879ee8e-9618-416c-8cae-c2820b9c1443\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/oft\", \"title\": \"Orthogonal Finetuning (OFT and BOFT)\", \"tokens\": 1680, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"422d6931e7d2f25de7bdf990eedef1ceb167fe303adaf96752e07e0437556830\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"221a9d58-845b-43ed-b85a-e3273632e65c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/oft\", \"title\": \"Orthogonal Finetuning (OFT and BOFT)\", \"tokens\": 1680, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"41cdedaf263ae4a0e3eb51a426a854d3c7c4f881b4348841e08ffc11d04a157f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3550, \"end_char_idx\": 6498, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "doc_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "ref_doc_id": "a879ee8e-9618-416c-8cae-c2820b9c1443", "chroma:document": "### BOFT-specific paramters\n\n`BOFTConfig` allows you to control how OFT/BOFT is applied to the base model through the following parameters:\n\n- `boft_block_size`: the BOFT matrix block size across different layers, expressed in `int`. Smaller block size results in sparser update matrices with fewer trainable paramters. **Note**, please choose `boft_block_size` to be divisible by most layer's input dimension (`in_features`), e.g., 4, 8, 16. Also, please only \nspecify either `boft_block_size` or `boft_block_num`, but not both simultaneously or leaving both to 0, because `boft_block_size` x `boft_block_num` must equal the layer's input dimension.\n- `boft_block_num`: the number of BOFT matrix blocks across different layers, expressed in `int`. Fewer blocks result in sparser update matrices with fewer trainable paramters. **Note**, please choose `boft_block_num` to be divisible by most layer's input dimension (`in_features`), e.g., 4, 8, 16. Also, please only \nspecify either `boft_block_size` or `boft_block_num`, but not both simultaneously or leaving both to 0, because `boft_block_size` x `boft_block_num` must equal the layer's input dimension.\n- `boft_n_butterfly_factor`: the number of butterfly factors. **Note**, for `boft_n_butterfly_factor=1`, BOFT is the same as vanilla OFT, for `boft_n_butterfly_factor=2`, the effective block size of OFT becomes twice as big and the number of blocks become half.\n- `bias`: specify if the `bias` parameters should be trained. Can be `\"none\"`, `\"all\"` or `\"boft_only\"`.\n- `boft_dropout`: specify the probability of multiplicative dropout.\n- `target_modules`: The modules (for example, attention blocks) to inject the OFT/BOFT matrices.\n- `modules_to_save`: List of modules apart from OFT/BOFT matrices to be set as trainable and saved in the final checkpoint. These typically include model's custom head that is randomly initialized for the fine-tuning task.\n\n\n\n## BOFT Example Usage\n\nFor an example of the BOFT method application to various downstream tasks, please refer to the following guides:\n\nTake a look at the following step-by-step guides on how to finetune a model with BOFT:\n- [Dreambooth finetuning with BOFT](../task_guides/boft_dreambooth) \n- [Controllable generation finetuning with BOFT (ControlNet)](../task_guides/boft_controlnet) \n\nFor the task of image classification, one can initialize the BOFT config for a DinoV2 model as follows:\n\n```py\nimport transformers\nfrom transformers import AutoModelForSeq2SeqLM, BOFTConfig\nfrom peft import BOFTConfig, get_peft_model\n\nconfig = BOFTConfig(\n    boft_block_size=4,\n    boft_n_butterfly_factor=2,\n    target_modules=[\"query\", \"value\", \"key\", \"output.dense\", \"mlp.fc1\", \"mlp.fc2\"],\n    boft_dropout=0.1,\n    bias=\"boft_only\",\n    modules_to_save=[\"classifier\"],\n)\n\nmodel = transformers.Dinov2ForImageClassification.from_pretrained(\n    \"facebook/dinov2-large\",\n    num_labels=100,\n)\n\nboft_model = get_peft_model(model, config)\n```"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 887a0b40-31aa-4c0f-92a3-5d3b55d19d1e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/adapter", "title": "Adapters", "tokens": 2443, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"887a0b40-31aa-4c0f-92a3-5d3b55d19d1e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"16f8d8fb-bb1a-4305-b0ad-b75971d46585\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"14524edaf3cc00e3ed2be165bfbc8e28de01af359f98b32f5bf21e6a0590b5f5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ebfeeb32-e94e-4871-b502-50ba46a2ad9f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"93d6e39db0b90135cd6352bf34776f5bc7fc09455f0c12c6a58aec21762dc9b5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3614, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "ref_doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "chroma:document": "# Adapters\n\nAdapter-based methods add extra trainable parameters after the attention and fully-connected layers of a frozen pretrained model to reduce memory-usage and speed up training. The method varies depending on the adapter, it could simply be an extra added layer or it could be expressing the weight updates \u2206W as a low-rank decomposition of the weight matrix. Either way, the adapters are typically small but demonstrate comparable performance to a fully finetuned model and enable training larger models with fewer resources.\n\nThis guide will give you a brief overview of the adapter methods supported by PEFT (if you're interested in learning more details about a specific method, take a look at the linked paper).\n\n## Low-Rank Adaptation (LoRA)\n\n<Tip>\n\nLoRA is one of the most popular PEFT methods and a good starting point if you're just getting started with PEFT. It was originally developed for large language models but it is a tremendously popular training method for diffusion models because of its efficiency and effectiveness.\n\n</Tip>\n\nAs mentioned briefly earlier, [LoRA](https://hf.co/papers/2106.09685) is a technique that accelerates finetuning large models while consuming less memory.\n\nLoRA represents the weight updates \u2206W with two smaller matrices (called *update matrices*) through low-rank decomposition. These new matrices can be trained to adapt to the new data while keeping the overall number of parameters low. The original weight matrix remains frozen and doesn't receive any further updates. To produce the final results, the original and extra adapted weights are combined. You could also merge the adapter weights with the base model to eliminate inference latency.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora_animated.gif\"/>\n</div>\n\nThis approach has a number of advantages:\n\n* LoRA makes finetuning more efficient by drastically reducing the number of trainable parameters.\n* The original pretrained weights are kept frozen, which means you can have multiple lightweight and portable LoRA models for various downstream tasks built on top of them.\n* LoRA is orthogonal to other parameter-efficient methods and can be combined with many of them.\n* Performance of models finetuned using LoRA is comparable to the performance of fully finetuned models.\n\nIn principle, LoRA can be applied to any subset of weight matrices in a neural network to reduce the number of trainable parameters. However, for simplicity and further parameter efficiency, LoRA is typically only applied to the attention blocks in Transformer models. The resulting number of trainable parameters in a LoRA model depends on the size of the update matrices, which is determined mainly by the rank `r` and the shape of the original weight matrix.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2103.10385\">Navigating Text-To-Image Customization: From LyCORIS Fine-Tuning to Model Evaluation</a></small>\n\n## Mixture of LoRA Experts (X-LoRA)\n\n[X-LoRA](https://arxiv.org/abs/2402.07148) is a mixture of experts method for LoRA which works by using dense or sparse gating to dynamically activate LoRA experts. The LoRA experts as well as the base model are frozen during training, resulting in a low parameter count as only the gating layers must be trained. In particular, the gating layers output scalings which (depending on config) are granular on the layer and token level."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: ebfeeb32-e94e-4871-b502-50ba46a2ad9f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/adapter", "title": "Adapters", "tokens": 2443, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"ebfeeb32-e94e-4871-b502-50ba46a2ad9f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"16f8d8fb-bb1a-4305-b0ad-b75971d46585\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"14524edaf3cc00e3ed2be165bfbc8e28de01af359f98b32f5bf21e6a0590b5f5\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"887a0b40-31aa-4c0f-92a3-5d3b55d19d1e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3fbf9c1eca5975aae93530fdf7816a4c84d4544830be81bd4574ae7023454ec0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"01d3e017-850f-4121-b691-a4525174642c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b8fa8e7e538c2946d1e95cae61f0ee99be79c58f9ffe96438777d1caf0426e27\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2008, \"end_char_idx\": 5580, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "ref_doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "chroma:document": "* The original pretrained weights are kept frozen, which means you can have multiple lightweight and portable LoRA models for various downstream tasks built on top of them.\n* LoRA is orthogonal to other parameter-efficient methods and can be combined with many of them.\n* Performance of models finetuned using LoRA is comparable to the performance of fully finetuned models.\n\nIn principle, LoRA can be applied to any subset of weight matrices in a neural network to reduce the number of trainable parameters. However, for simplicity and further parameter efficiency, LoRA is typically only applied to the attention blocks in Transformer models. The resulting number of trainable parameters in a LoRA model depends on the size of the update matrices, which is determined mainly by the rank `r` and the shape of the original weight matrix.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2103.10385\">Navigating Text-To-Image Customization: From LyCORIS Fine-Tuning to Model Evaluation</a></small>\n\n## Mixture of LoRA Experts (X-LoRA)\n\n[X-LoRA](https://arxiv.org/abs/2402.07148) is a mixture of experts method for LoRA which works by using dense or sparse gating to dynamically activate LoRA experts. The LoRA experts as well as the base model are frozen during training, resulting in a low parameter count as only the gating layers must be trained. In particular, the gating layers output scalings which (depending on config) are granular on the layer and token level. Additionally, during inference, X-LoRA dynamically activates LoRA adapters to recall knowledge and effectively mix them:\n\nThe below graphic demonstrates how the scalings change for different prompts for each token. This highlights the activation of different adapters as the generation progresses and the sequence creates new context.\n\n![Token-by-token scalings](https://github.com/EricLBuehler/xlora/raw/master/res/token_by_token_scalings.gif)\n\nFor each step, X-LoRA requires the base model to be run twice: first, to get hidden states without any LoRA adapters, and secondly, the hidden states are used to calculate scalings which are applied to the LoRA adapters and the model is run a second time. The output of the second run is the result of the model step.\n\nUltimately, X-LoRA allows the model to reflect upon it's knowledge because of the dual forward pass scheme, and dynamically reconfigure the architecture.\n\n## Low-Rank Hadamard Product (LoHa)\n\nLow-rank decomposition can impact performance because the weight updates are limited to the low-rank space, which can constrain a model's expressiveness. However, you don't necessarily want to use a larger rank because it increases the number of trainable parameters. To address this, [LoHa](https://huggingface.co/papers/2108.06098) (a method originally developed for computer vision) was applied to diffusion models where the ability to generate diverse images is an important consideration. LoHa should also work with general model types, but the embedding layers aren't currently implemented in PEFT.\n\nLoHa uses the [Hadamard product](https://en.wikipedia.org/wiki/Hadamard_product_(matrices)) (element-wise product) instead of the matrix product. \u2206W is represented by four smaller matrices instead of two - like in LoRA - and each pair of these low-rank matrices are combined with the Hadamard product. As a result, \u2206W can have the same number of trainable parameters but a higher rank and expressivity."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 01d3e017-850f-4121-b691-a4525174642c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/adapter", "title": "Adapters", "tokens": 2443, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"01d3e017-850f-4121-b691-a4525174642c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"16f8d8fb-bb1a-4305-b0ad-b75971d46585\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"14524edaf3cc00e3ed2be165bfbc8e28de01af359f98b32f5bf21e6a0590b5f5\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ebfeeb32-e94e-4871-b502-50ba46a2ad9f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"e06b0b77e84a140b56aaa5b52f76d5b839c975b35820f9a8bf6e5a3df9ae9a51\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d5f81d2b-0ea9-4461-981a-cde7807de092\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ded1b25828766b368185f7ac6281807cbf723ed6dd1a71527c7168bef68bdd62\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3830, \"end_char_idx\": 7307, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "ref_doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "chroma:document": "This highlights the activation of different adapters as the generation progresses and the sequence creates new context.\n\n![Token-by-token scalings](https://github.com/EricLBuehler/xlora/raw/master/res/token_by_token_scalings.gif)\n\nFor each step, X-LoRA requires the base model to be run twice: first, to get hidden states without any LoRA adapters, and secondly, the hidden states are used to calculate scalings which are applied to the LoRA adapters and the model is run a second time. The output of the second run is the result of the model step.\n\nUltimately, X-LoRA allows the model to reflect upon it's knowledge because of the dual forward pass scheme, and dynamically reconfigure the architecture.\n\n## Low-Rank Hadamard Product (LoHa)\n\nLow-rank decomposition can impact performance because the weight updates are limited to the low-rank space, which can constrain a model's expressiveness. However, you don't necessarily want to use a larger rank because it increases the number of trainable parameters. To address this, [LoHa](https://huggingface.co/papers/2108.06098) (a method originally developed for computer vision) was applied to diffusion models where the ability to generate diverse images is an important consideration. LoHa should also work with general model types, but the embedding layers aren't currently implemented in PEFT.\n\nLoHa uses the [Hadamard product](https://en.wikipedia.org/wiki/Hadamard_product_(matrices)) (element-wise product) instead of the matrix product. \u2206W is represented by four smaller matrices instead of two - like in LoRA - and each pair of these low-rank matrices are combined with the Hadamard product. As a result, \u2206W can have the same number of trainable parameters but a higher rank and expressivity.\n\n## Low-Rank Kronecker Product (LoKr)\n\n[LoKr](https://hf.co/papers/2309.14859) is very similar to LoRA and LoHa, and it is also mainly applied to diffusion models, though you could also use it with other model types. LoKr replaces the matrix product with the [Kronecker product](https://en.wikipedia.org/wiki/Kronecker_product) instead. The Kronecker product decomposition creates a block matrix which preserves the rank of the original weight matrix. Another benefit of the Kronecker product is that it can be vectorized by stacking the matrix columns. This can speed up the process because you're avoiding fully reconstructing \u2206W.\n\n## Orthogonal Finetuning (OFT)\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/oft.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2306.07280\">Controlling Text-to-Image Diffusion by Orthogonal Finetuning</a></small>\n\n[OFT](https://hf.co/papers/2306.07280) is a method that primarily focuses on preserving a pretrained model's generative performance in the finetuned model. It tries to maintain the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer because this better captures the semantic information among neurons. This means OFT is more capable at preserving the subject and it is better for controllable generation (similar to [ControlNet](https://huggingface.co/docs/diffusers/using-diffusers/controlnet)).\n\nOFT preserves the hyperspherical energy by learning an orthogonal transformation for neurons to keep the cosine similarity between them unchanged. In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d5f81d2b-0ea9-4461-981a-cde7807de092
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/adapter", "title": "Adapters", "tokens": 2443, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d5f81d2b-0ea9-4461-981a-cde7807de092\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"16f8d8fb-bb1a-4305-b0ad-b75971d46585\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"14524edaf3cc00e3ed2be165bfbc8e28de01af359f98b32f5bf21e6a0590b5f5\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"01d3e017-850f-4121-b691-a4525174642c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"2bf4a7892c684c9641febee39934326ae6998310a9c529798be77b3ed75fb114\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"81119ec2-6d2e-4d18-a72f-234c3dee6fc5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d22305c2567f2a1b668d2f4033b6d3a3ea363ab2002ce73ce1b774ae42aa2269\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5582, \"end_char_idx\": 9111, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "ref_doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "chroma:document": "## Low-Rank Kronecker Product (LoKr)\n\n[LoKr](https://hf.co/papers/2309.14859) is very similar to LoRA and LoHa, and it is also mainly applied to diffusion models, though you could also use it with other model types. LoKr replaces the matrix product with the [Kronecker product](https://en.wikipedia.org/wiki/Kronecker_product) instead. The Kronecker product decomposition creates a block matrix which preserves the rank of the original weight matrix. Another benefit of the Kronecker product is that it can be vectorized by stacking the matrix columns. This can speed up the process because you're avoiding fully reconstructing \u2206W.\n\n## Orthogonal Finetuning (OFT)\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/oft.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2306.07280\">Controlling Text-to-Image Diffusion by Orthogonal Finetuning</a></small>\n\n[OFT](https://hf.co/papers/2306.07280) is a method that primarily focuses on preserving a pretrained model's generative performance in the finetuned model. It tries to maintain the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer because this better captures the semantic information among neurons. This means OFT is more capable at preserving the subject and it is better for controllable generation (similar to [ControlNet](https://huggingface.co/docs/diffusers/using-diffusers/controlnet)).\n\nOFT preserves the hyperspherical energy by learning an orthogonal transformation for neurons to keep the cosine similarity between them unchanged. In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix. However, to be parameter-efficient, the orthogonal matrix is represented as a block-diagonal matrix with rank `r` blocks. Whereas LoRA reduces the number of trainable parameters with low-rank structures, OFT reduces the number of trainable parameters with a sparse block-diagonal matrix structure.\n\n## Orthogonal Butterfly (BOFT)\n\n[BOFT](https://hf.co/papers/2311.06243) is a method that primarily focuses on preserving a pretrained model's generative performance in the finetuned model. It tries to maintain the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer because this better captures the semantic information among neurons. This means OFT is more capable at preserving the subject and it is better for controllable generation (similar to [ControlNet](https://huggingface.co/docs/diffusers/using-diffusers/controlnet)).\n\nOFT preserves the hyperspherical energy by learning an orthogonal transformation for neurons to keep the cosine similarity between them unchanged. In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix. However, to be parameter-efficient, the orthogonal matrix is represented as a block-diagonal matrix with rank `r` blocks. Whereas LoRA reduces the number of trainable parameters with low-rank structures, OFT reduces the number of trainable parameters with a sparse block-diagonal matrix structure.\n\n## Adaptive Low-Rank Adaptation (AdaLoRA)\n\n[AdaLoRA](https://hf.co/papers/2303.10512) manages the parameter budget introduced from LoRA by allocating more parameters - in other words, a higher rank `r` - for important weight matrices that are better adapted for a task and pruning less important ones. The rank is controlled by a method similar to singular value decomposition (SVD)."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 81119ec2-6d2e-4d18-a72f-234c3dee6fc5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/adapter", "title": "Adapters", "tokens": 2443, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"81119ec2-6d2e-4d18-a72f-234c3dee6fc5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"16f8d8fb-bb1a-4305-b0ad-b75971d46585\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"14524edaf3cc00e3ed2be165bfbc8e28de01af359f98b32f5bf21e6a0590b5f5\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d5f81d2b-0ea9-4461-981a-cde7807de092\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/adapter\", \"title\": \"Adapters\", \"tokens\": 2443, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"98c6ada4ab0caa5869155e808a28e5b723d5df34cdc1573c59b620b9dba456d4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7199, \"end_char_idx\": 10834, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "ref_doc_id": "16f8d8fb-bb1a-4305-b0ad-b75971d46585", "chroma:document": "In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix. However, to be parameter-efficient, the orthogonal matrix is represented as a block-diagonal matrix with rank `r` blocks. Whereas LoRA reduces the number of trainable parameters with low-rank structures, OFT reduces the number of trainable parameters with a sparse block-diagonal matrix structure.\n\n## Orthogonal Butterfly (BOFT)\n\n[BOFT](https://hf.co/papers/2311.06243) is a method that primarily focuses on preserving a pretrained model's generative performance in the finetuned model. It tries to maintain the same cosine similarity (hyperspherical energy) between all pairwise neurons in a layer because this better captures the semantic information among neurons. This means OFT is more capable at preserving the subject and it is better for controllable generation (similar to [ControlNet](https://huggingface.co/docs/diffusers/using-diffusers/controlnet)).\n\nOFT preserves the hyperspherical energy by learning an orthogonal transformation for neurons to keep the cosine similarity between them unchanged. In practice, this means taking the matrix product of an orthogonal matrix with the pretrained weight matrix. However, to be parameter-efficient, the orthogonal matrix is represented as a block-diagonal matrix with rank `r` blocks. Whereas LoRA reduces the number of trainable parameters with low-rank structures, OFT reduces the number of trainable parameters with a sparse block-diagonal matrix structure.\n\n## Adaptive Low-Rank Adaptation (AdaLoRA)\n\n[AdaLoRA](https://hf.co/papers/2303.10512) manages the parameter budget introduced from LoRA by allocating more parameters - in other words, a higher rank `r` - for important weight matrices that are better adapted for a task and pruning less important ones. The rank is controlled by a method similar to singular value decomposition (SVD). The \u2206W is parameterized with two orthogonal matrices and a diagonal matrix which contains singular values. This parametrization method avoids iteratively applying SVD which is computationally expensive. Based on this method, the rank of \u2206W is adjusted according to an importance score. \u2206W is divided into triplets and each triplet is scored according to its contribution to model performance. Triplets with low importance scores are pruned and triplets with high importance scores are kept for finetuning.\n\n## Llama-Adapter\n\n[Llama-Adapter](https://hf.co/papers/2303.16199) is a method for adapting Llama into a instruction-following model. To help adapt the model for instruction-following, the adapter is trained with a 52K instruction-output dataset.\n\nA set of of learnable adaption prompts are prefixed to the input instruction tokens. These are inserted into the upper layers of the model because it is better to learn with the higher-level semantics of the pretrained model. The instruction-output tokens prefixed to the input guide the adaption prompt to generate a contextual response.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/llama-adapter.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2303.16199\">LLaMA-Adapter: Efficient Fine-tuning of Language Models with Zero-init Attention</a></small>\n\nTo avoid adding noise to the tokens, the adapter uses zero-initialized attention. On top of this, the adapter adds a learnable gating factor (initialized with zeros) to progressively add information to the model during training. This prevents overwhelming the model's pretrained knowledge with the newly learned instructions."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: ab7246b8-7d3e-4482-a603-6fbbf68fc1db
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/prompting", "title": "Soft prompts", "tokens": 1568, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"ab7246b8-7d3e-4482-a603-6fbbf68fc1db\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"cea0e352-2d07-4b4f-8bec-05ba1ad0865e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"9ced6c306fe65a64f13a8b54c6b09cdadc0dc9194bcdb8a53ed2d63c25163117\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"acbd87f0-83f6-450b-b786-103a339ca191\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"393ae6aa986ae952811a898e9002e7d5bec13b9e76da120c93466e145bf57cd3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3590, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "ref_doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Soft prompts\n\nTraining large pretrained language models is very time-consuming and compute-intensive. As they continue to grow in size, there is increasing interest in more efficient training methods such as *prompting*. Prompting primes a frozen pretrained model for a specific downstream task by including a text prompt that describes the task or even demonstrates an example of the task. With prompting, you can avoid fully training a separate model for each downstream task, and use the same frozen pretrained model instead. This is a lot easier because you can use the same model for several different tasks, and it is significantly more efficient to train and store a smaller set of prompt parameters than to train all the model's parameters.\n\nThere are two categories of prompting methods:\n\n- hard prompts are manually handcrafted text prompts with discrete input tokens; the downside is that it requires a lot of effort to create a good prompt\n- soft prompts are learnable tensors concatenated with the input embeddings that can be optimized to a dataset; the downside is that they aren't human readable because you aren't matching these \"virtual tokens\" to the embeddings of a real word\n\nThis conceptual guide provides a brief overview of the soft prompt methods included in \ud83e\udd17 PEFT: prompt tuning, prefix tuning, P-tuning, and multitask prompt tuning.\n\n## Prompt tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/prompt-tuning.png\"/>\n</div>\n<small>Only train and store a significantly smaller set of task-specific prompt parameters <a href=\"https://hf.co/papers/2104.08691\">(image source)</a>.</small>\n\n[Prompt tuning](https://hf.co/papers/2104.08691) was developed for text classification tasks on T5 models, and all downstream tasks are cast as a text generation task. For example, sequence classification usually assigns a single class label to a sequence of text. By casting it as a text generation task, the tokens that make up the class label are *generated*. Prompts are added to the input as a series of tokens. Typically, the model parameters are fixed which means the prompt tokens are also fixed by the model parameters.\n\nThe key idea behind prompt tuning is that prompt tokens have their own parameters that are updated independently. This means you can keep the pretrained model's parameters frozen, and only update the gradients of the prompt token embeddings. The results are comparable to the traditional method of training the entire model, and prompt tuning performance scales as model size increases.\n\nTake a look at [Prompt tuning for causal language modeling](../task_guides/clm-prompt-tuning) for a step-by-step guide on how to train a model with prompt tuning.\n\n## Prefix tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/prefix-tuning.png\"/>\n</div>\n<small>Optimize the prefix parameters for each task <a href=\"https://hf.co/papers/2101.00190\">(image source)</a>.</small>\n\n[Prefix tuning](https://hf.co/papers/2101.00190) was designed for natural language generation (NLG) tasks on GPT models. It is very similar to prompt tuning; prefix tuning also prepends a sequence of task-specific vectors to the input that can be trained and updated while keeping the rest of the pretrained model's parameters frozen."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: acbd87f0-83f6-450b-b786-103a339ca191
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/prompting", "title": "Soft prompts", "tokens": 1568, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"acbd87f0-83f6-450b-b786-103a339ca191\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"cea0e352-2d07-4b4f-8bec-05ba1ad0865e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"9ced6c306fe65a64f13a8b54c6b09cdadc0dc9194bcdb8a53ed2d63c25163117\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ab7246b8-7d3e-4482-a603-6fbbf68fc1db\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"93e9d07f9c2501a89a13f4b7a9a8a116db36b1b7b62f99f73a0c7d050f4cf8fb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6d225d4e-9a0b-421b-99aa-947a03358a2f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2d965caa696bebd52d4fde8cf0bf98d8ea7fba47756c6c41468deff0fe0502f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2042, \"end_char_idx\": 5660, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "ref_doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "chroma:document": "For example, sequence classification usually assigns a single class label to a sequence of text. By casting it as a text generation task, the tokens that make up the class label are *generated*. Prompts are added to the input as a series of tokens. Typically, the model parameters are fixed which means the prompt tokens are also fixed by the model parameters.\n\nThe key idea behind prompt tuning is that prompt tokens have their own parameters that are updated independently. This means you can keep the pretrained model's parameters frozen, and only update the gradients of the prompt token embeddings. The results are comparable to the traditional method of training the entire model, and prompt tuning performance scales as model size increases.\n\nTake a look at [Prompt tuning for causal language modeling](../task_guides/clm-prompt-tuning) for a step-by-step guide on how to train a model with prompt tuning.\n\n## Prefix tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/prefix-tuning.png\"/>\n</div>\n<small>Optimize the prefix parameters for each task <a href=\"https://hf.co/papers/2101.00190\">(image source)</a>.</small>\n\n[Prefix tuning](https://hf.co/papers/2101.00190) was designed for natural language generation (NLG) tasks on GPT models. It is very similar to prompt tuning; prefix tuning also prepends a sequence of task-specific vectors to the input that can be trained and updated while keeping the rest of the pretrained model's parameters frozen. \n\nThe main difference is that the prefix parameters are inserted in **all** of the model layers, whereas prompt tuning only adds the prompt parameters to the model input embeddings. The prefix parameters are also optimized by a separate feed-forward network (FFN) instead of training directly on the soft prompts because it causes instability and hurts performance. The FFN is discarded after updating the soft prompts.\n\nAs a result, the authors found that prefix tuning demonstrates comparable performance to fully finetuning a model, despite having 1000x fewer parameters, and it performs even better in low-data settings.\n\nTake a look at [Prefix tuning for conditional generation](../task_guides/seq2seq-prefix-tuning) for a step-by-step guide on how to train a model with prefix tuning.\n\n## P-tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/p-tuning.png\"/>\n</div>\n<small>Prompt tokens can be inserted anywhere in the input sequence, and they are optimized by a prompt encoder <a href=\"https://hf.co/papers/2103.10385\">(image source)</a>.</small>\n\n[P-tuning](https://hf.co/papers/2103.10385) is designed for natural language understanding (NLU) tasks and all language models. \nIt is another variation of a soft prompt method; P-tuning also adds a trainable embedding tensor that can be optimized to find better prompts, and it uses a prompt encoder (a bidirectional long-short term memory network or LSTM) to optimize the prompt parameters. Unlike prefix tuning though:\n\n- the prompt tokens can be inserted anywhere in the input sequence, and it isn't restricted to only the beginning\n- the prompt tokens are only added to the input instead of adding them to every layer of the model\n- introducing *anchor* tokens can improve performance because they indicate characteristics of a component in the input sequence\n\nThe results suggest that P-tuning is more efficient than manually crafting prompts, and it enables GPT-like models to compete with BERT-like models on NLU tasks."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 6d225d4e-9a0b-421b-99aa-947a03358a2f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/conceptual_guides/prompting", "title": "Soft prompts", "tokens": 1568, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"6d225d4e-9a0b-421b-99aa-947a03358a2f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"cea0e352-2d07-4b4f-8bec-05ba1ad0865e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"9ced6c306fe65a64f13a8b54c6b09cdadc0dc9194bcdb8a53ed2d63c25163117\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"acbd87f0-83f6-450b-b786-103a339ca191\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/conceptual_guides/prompting\", \"title\": \"Soft prompts\", \"tokens\": 1568, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f2af67c8bb0bed7d4a8d6b2714f084cf6297ae67d2dea0c05e12fba580f873cb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3957, \"end_char_idx\": 7365, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "ref_doc_id": "cea0e352-2d07-4b4f-8bec-05ba1ad0865e", "chroma:document": "The FFN is discarded after updating the soft prompts.\n\nAs a result, the authors found that prefix tuning demonstrates comparable performance to fully finetuning a model, despite having 1000x fewer parameters, and it performs even better in low-data settings.\n\nTake a look at [Prefix tuning for conditional generation](../task_guides/seq2seq-prefix-tuning) for a step-by-step guide on how to train a model with prefix tuning.\n\n## P-tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/p-tuning.png\"/>\n</div>\n<small>Prompt tokens can be inserted anywhere in the input sequence, and they are optimized by a prompt encoder <a href=\"https://hf.co/papers/2103.10385\">(image source)</a>.</small>\n\n[P-tuning](https://hf.co/papers/2103.10385) is designed for natural language understanding (NLU) tasks and all language models. \nIt is another variation of a soft prompt method; P-tuning also adds a trainable embedding tensor that can be optimized to find better prompts, and it uses a prompt encoder (a bidirectional long-short term memory network or LSTM) to optimize the prompt parameters. Unlike prefix tuning though:\n\n- the prompt tokens can be inserted anywhere in the input sequence, and it isn't restricted to only the beginning\n- the prompt tokens are only added to the input instead of adding them to every layer of the model\n- introducing *anchor* tokens can improve performance because they indicate characteristics of a component in the input sequence\n\nThe results suggest that P-tuning is more efficient than manually crafting prompts, and it enables GPT-like models to compete with BERT-like models on NLU tasks.\n\nTake a look at [P-tuning for sequence classification](../task_guides/ptuning-seq-classification) for a step-by-step guide on how to train a model with P-tuning.\n\n## Multitask prompt tuning\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/mpt.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2303.02861\">Multitask prompt tuning enables parameter-efficient transfer learning</a>.</small>\n\n[Multitask prompt tuning (MPT)](https://hf.co/papers/2303.02861) learns a single prompt from data for multiple task types that can be shared for different target tasks. Other existing approaches learn a separate soft prompt for each task that need to be retrieved or aggregated for adaptation to target tasks. MPT consists of two stages:\n\n1. source training - for each task, its soft prompt is decomposed into task-specific vectors. The task-specific vectors are multiplied together to form another matrix W, and the Hadamard product is used between W and a shared prompt matrix P to generate a task-specific prompt matrix. The task-specific prompts are distilled into a single prompt matrix that is shared across all tasks. This prompt is trained with multitask training.\n2. target adaptation - to adapt the single prompt for a target task, a target prompt is initialized and expressed as the Hadamard product of the shared prompt matrix and the task-specific low-rank prompt matrix.\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/mpt-decomposition.png\"/>\n</div>\n<small><a href=\"https://hf.co/papers/2103.10385\">Prompt decomposition</a>.</small>"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 25ee240a-0520-4a72-bea0-5ee70d4bd94d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"25ee240a-0520-4a72-bea0-5ee70d4bd94d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"86808eef-53d0-4bac-9b43-429aeeae6d2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4e22b3bed5a7d9a131acbdcbdc4860fd36966da825220196afd9b28bc29815ce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3151, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# DeepSpeed\n\n[DeepSpeed](https://www.deepspeed.ai/) is a library designed for speed and scale for distributed training of large models with billions of parameters. At its core is the Zero Redundancy Optimizer (ZeRO) that shards optimizer states (ZeRO-1), gradients (ZeRO-2), and parameters (ZeRO-3) across data parallel processes. This drastically reduces memory usage, allowing you to scale your training to billion parameter models. To unlock even more memory efficiency, ZeRO-Offload reduces GPU compute and memory by leveraging CPU resources during optimization.\n\nBoth of these features are supported in \ud83e\udd17 Accelerate, and you can use them with \ud83e\udd17 PEFT. \n\n## Compatibility with `bitsandbytes` quantization + LoRA\n\nBelow is a table that summarizes the compatibility between PEFT's LoRA, [`bitsandbytes`](https://github.com/TimDettmers/bitsandbytes) library and DeepSpeed Zero stages with respect to fine-tuning. DeepSpeed Zero-1 and 2 will have no effect at inference as stage 1 shards the optimizer states and stage 2 shards the optimizer states and gradients:\n\n| DeepSpeed stage   | Is compatible? |\n|---|---|\n| Zero-1 |  \ud83d\udfe2 |\n| Zero-2   |  \ud83d\udfe2 |\n| Zero-3  |  \ud83d\udfe2 |\n\nFor DeepSpeed Stage 3 + QLoRA, please refer to the section [Use PEFT QLoRA and DeepSpeed with ZeRO3 for finetuning large models on multiple GPUs](#use-peft-qlora-and-deepspeed-with-zero3-for-finetuning-large-models-on-multiple-gpus) below.\n\nFor confirming these observations, we ran the SFT (Supervised Fine-tuning) [offical example scripts](https://github.com/huggingface/trl/tree/main/examples) of the [Transformers Reinforcement Learning (TRL) library](https://github.com/huggingface/trl) using QLoRA + PEFT and the accelerate configs available [here](https://github.com/huggingface/trl/tree/main/examples/accelerate_configs). We ran these experiments on a 2x NVIDIA T4 GPU.\n\n# Use PEFT and DeepSpeed with ZeRO3 for finetuning large models on multiple devices and multiple nodes\n\nThis section of guide will help you learn how to use our DeepSpeed [training script](https://github.com/huggingface/peft/blob/main/examples/sft/train.py) for performing SFT. You'll configure the script to do SFT (supervised fine-tuning) of Llama-70B model with LoRA and ZeRO-3 on 8xH100 80GB GPUs on a single machine. You can configure it to scale to multiple machines by changing the accelerate config.\n\n## Configuration\n\nStart by running the following command to [create a DeepSpeed configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file deepspeed_config.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 86808eef-53d0-4bac-9b43-429aeeae6d2c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"86808eef-53d0-4bac-9b43-429aeeae6d2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"25ee240a-0520-4a72-bea0-5ee70d4bd94d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"7e9d8060314e14318b38514bd7ef2d363471ab2e6e8544fc2fe9b996941f5d4d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"089dae0b-bc31-42db-bfd7-bcab4c209c6b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bab63b5f55f768fe8f5ce85aa6f09a1caba586dcac2512924762bfd435a35d4b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1577, \"end_char_idx\": 4587, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "For confirming these observations, we ran the SFT (Supervised Fine-tuning) [offical example scripts](https://github.com/huggingface/trl/tree/main/examples) of the [Transformers Reinforcement Learning (TRL) library](https://github.com/huggingface/trl) using QLoRA + PEFT and the accelerate configs available [here](https://github.com/huggingface/trl/tree/main/examples/accelerate_configs). We ran these experiments on a 2x NVIDIA T4 GPU.\n\n# Use PEFT and DeepSpeed with ZeRO3 for finetuning large models on multiple devices and multiple nodes\n\nThis section of guide will help you learn how to use our DeepSpeed [training script](https://github.com/huggingface/peft/blob/main/examples/sft/train.py) for performing SFT. You'll configure the script to do SFT (supervised fine-tuning) of Llama-70B model with LoRA and ZeRO-3 on 8xH100 80GB GPUs on a single machine. You can configure it to scale to multiple machines by changing the accelerate config.\n\n## Configuration\n\nStart by running the following command to [create a DeepSpeed configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file deepspeed_config.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll use ZeRO-3 so make sure you pick those options.\n\n```bash\n`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them. Pass the same value as you would pass via cmd argument else you will encounter mismatch error.\n`gradient_clipping`: Enable gradient clipping with value. Don't set this as you will be passing it via cmd arguments.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO >= Stage-2. Set this as `none` as don't want to enable offloading.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3. Set this as `none` as don't want to enable offloading.\n`zero3_init_flag`: Decides whether to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with ZeRO Stage-3. Set this to `True`.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3. Set this to `True`.\n`mixed_precision`: `no` for FP32 training, `fp16` for FP16 mixed-precision training and `bf16` for BF16 mixed-precision training. Set this to `True`."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 089dae0b-bc31-42db-bfd7-bcab4c209c6b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"089dae0b-bc31-42db-bfd7-bcab4c209c6b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"86808eef-53d0-4bac-9b43-429aeeae6d2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"2d0627e65740cf39ed16a9fcefe119f934763804ad46f57d6cc9135768750573\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ef963d88-e0c9-4632-a69c-96fe6cee5c24\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"90cbde9599a974dc385108ddc233a509fee79f90aa00e2e3fdffa8577ceebf8f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2996, \"end_char_idx\": 6882, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "```bash\naccelerate config --config_file deepspeed_config.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll use ZeRO-3 so make sure you pick those options.\n\n```bash\n`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them. Pass the same value as you would pass via cmd argument else you will encounter mismatch error.\n`gradient_clipping`: Enable gradient clipping with value. Don't set this as you will be passing it via cmd arguments.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO >= Stage-2. Set this as `none` as don't want to enable offloading.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3. Set this as `none` as don't want to enable offloading.\n`zero3_init_flag`: Decides whether to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with ZeRO Stage-3. Set this to `True`.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3. Set this to `True`.\n`mixed_precision`: `no` for FP32 training, `fp16` for FP16 mixed-precision training and `bf16` for BF16 mixed-precision training. Set this to `True`.\n```\n\nOnce this is done, the corresponding config should look like below and you can find it in config folder at [deepspeed_config.yaml](https://github.com/huggingface/peft/blob/main/examples/sft/configs/deepspeed_config.yaml):\n\n```yml\ncompute_environment: LOCAL_MACHINE                                                                                                                                           \ndebug: false\ndeepspeed_config:\n  deepspeed_multinode_launcher: standard\n  gradient_accumulation_steps: 4\n  offload_optimizer_device: none\n  offload_param_device: none\n  zero3_init_flag: true\n  zero3_save_16bit_model: true\n  zero_stage: 3\ndistributed_type: DEEPSPEED\ndowncast_bf16: 'no'\nmachine_rank: 0\nmain_training_function: main\nmixed_precision: bf16\nnum_machines: 1\nnum_processes: 8\nrdzv_backend: static\nsame_network: true\ntpu_env: []\ntpu_use_cluster: false\ntpu_use_sudo: false\nuse_cpu: false\n```\n\n## Launch command\n\nThe launch command is available at [run_peft_deepspeed.sh](https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_deepspeed.sh) and it is also shown below:\n```bash\naccelerate launch --config_file \"configs/deepspeed_config.yaml\"  train.py \\\n--seed 100 \\\n--model_name_or_path \"meta-llama/Llama-2-70b-hf\" \\\n--dataset_name \"smangrul/ultrachat-10k-chatml\" \\\n--chat_template_format \"chatml\" \\\n--add_special_tokens False \\\n--append_concat_token False \\\n--splits \"train,test\" \\\n--max_seq_len 2048 \\\n--num_train_epochs 1 \\\n--logging_steps 5 \\\n--log_level \"info\" \\\n--logging_strategy \"steps\" \\\n--evaluation_strategy \"epoch\" \\\n--save_strategy \"epoch\" \\\n--push_to_hub \\\n--hub_private_repo True \\\n--hub_strategy \"every_save\" \\\n--bf16 True \\\n--packing True \\\n--learning_rate 1e-4 \\\n--lr_scheduler_type \"cosine\" \\\n--weight_decay 1e-4 \\\n--warmup_ratio 0.0 \\\n--max_grad_norm 1.0 \\\n--output_dir \"llama-sft-lora-deepspeed\" \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--gradient_accumulation_steps 4 \\\n--gradient_checkpointing True \\\n--use_reentrant False \\\n--dataset_text_field \"content\" \\\n--use_flash_attn True \\\n--use_peft_lora True \\\n--lora_r 8 \\\n--lora_alpha 16 \\\n--lora_dropout 0.1 \\\n--lora_target_modules \"all-linear\" \\\n--use_4bit_quantization False\n```\n\nNotice that we are using LoRA with  rank=8, alpha=16 and targeting all linear layers."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: ef963d88-e0c9-4632-a69c-96fe6cee5c24
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"ef963d88-e0c9-4632-a69c-96fe6cee5c24\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"089dae0b-bc31-42db-bfd7-bcab4c209c6b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"331f4be617b173d0f88190a512cffddce6964aaa28ee813d7af9af2f5d407098\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6c70689e-ed8c-42ab-a308-9ee88c6ca160\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"846e2a44cdc7b16e37a7704e3e55d9cba0efca8a882f1be3b6654002d2e375b9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6883, \"end_char_idx\": 9379, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "We are passing the deepspeed config file and finetuning 70B Llama model on a subset of the ultrachat dataset.\n\n## The important parts\n\nLet's dive a little deeper into the script so you can see what's going on, and understand how it works.\n\nThe first thing to know is that the script uses DeepSpeed for distributed training as the DeepSpeed config has been passed. The `SFTTrainer` class handles all the heavy lifting of creating the PEFT model using the peft config that is passed. After that, when you call `trainer.train()`, `SFTTrainer` internally uses \ud83e\udd17 Accelerate to prepare the model, optimizer and trainer using the DeepSpeed config to create DeepSpeed engine which is then trained. The main code snippet is below:\n\n```python\n# trainer\ntrainer = SFTTrainer(\n    model=model,\n    tokenizer=tokenizer,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n    peft_config=peft_config,\n    packing=data_args.packing,\n    dataset_kwargs={\n        \"append_concat_token\": data_args.append_concat_token,\n        \"add_special_tokens\": data_args.add_special_tokens,\n    },\n    dataset_text_field=data_args.dataset_text_field,\n    max_seq_length=data_args.max_seq_length,\n)\ntrainer.accelerator.print(f\"{trainer.model}\")\n\n# train\ncheckpoint = None\nif training_args.resume_from_checkpoint is not None:\n    checkpoint = training_args.resume_from_checkpoint\ntrainer.train(resume_from_checkpoint=checkpoint)\n\n# saving final model\ntrainer.save_model()\n```\n\n## Memory usage\n\nIn the above example, the memory consumed per GPU is 64 GB (80%) as seen in the screenshot below:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_deepspeed_mem_usage.png\"/>\n</div>\n<small>GPU memory usage for the training run</small>\n\n## More resources\nYou can also refer this blog post [Falcon 180B Finetuning using \ud83e\udd17 PEFT and DeepSpeed](https://medium.com/@sourabmangrulkar/falcon-180b-finetuning-using-peft-and-deepspeed-b92643091d99) on how to finetune 180B Falcon model on 16 A100 GPUs on 2 machines.\n\n\n# Use PEFT QLoRA and DeepSpeed with ZeRO3 for finetuning large models on multiple GPUs\n\nIn this section, we will look at how to use QLoRA and DeepSpeed Stage-3 for finetuning 70B llama model on 2X40GB GPUs.\nFor this, we first need `bitsandbytes>=0.43.0`, `accelerate>=0.28.0`, `transformers>4.38.2`, `trl>0.7.11` and `peft>0.9.0`. We need to set `zero3_init_flag` to true when using Accelerate config."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 6c70689e-ed8c-42ab-a308-9ee88c6ca160
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"6c70689e-ed8c-42ab-a308-9ee88c6ca160\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ef963d88-e0c9-4632-a69c-96fe6cee5c24\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"c5c5e93ba5a37a9a519fc6aff103784203e97c4216ee520362dc0937b6505f5d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1a8bf6d1-86bf-46c7-af85-2a66b11f5e90\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"137acbe3d5a648966d550c6e35f4b4a9931f0520677e3bf75f773b1f468e49e4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8978, \"end_char_idx\": 11724, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "# Use PEFT QLoRA and DeepSpeed with ZeRO3 for finetuning large models on multiple GPUs\n\nIn this section, we will look at how to use QLoRA and DeepSpeed Stage-3 for finetuning 70B llama model on 2X40GB GPUs.\nFor this, we first need `bitsandbytes>=0.43.0`, `accelerate>=0.28.0`, `transformers>4.38.2`, `trl>0.7.11` and `peft>0.9.0`. We need to set `zero3_init_flag` to true when using Accelerate config. Below is the config which can be found at [deepspeed_config_z3_qlora.yaml](https://github.com/huggingface/peft/blob/main/examples/sft/configs/deepspeed_config_z3_qlora.yaml):\n\n```yml\ncompute_environment: LOCAL_MACHINE                                                                                                                                           \ndebug: false\ndeepspeed_config:\n  deepspeed_multinode_launcher: standard\n  offload_optimizer_device: none\n  offload_param_device: none\n  zero3_init_flag: true\n  zero3_save_16bit_model: true\n  zero_stage: 3\ndistributed_type: DEEPSPEED\ndowncast_bf16: 'no'\nmachine_rank: 0\nmain_training_function: main\nmixed_precision: bf16\nnum_machines: 1\nnum_processes: 2\nrdzv_backend: static\nsame_network: true\ntpu_env: []\ntpu_use_cluster: false\ntpu_use_sudo: false\nuse_cpu: false\n```\n\nLaunch command is given below which is available at [run_peft_qlora_deepspeed_stage3.sh](https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_deepspeed.sh):\n```\naccelerate launch --config_file \"configs/deepspeed_config_z3_qlora.yaml\"  train.py \\\n--seed 100 \\\n--model_name_or_path \"meta-llama/Llama-2-70b-hf\" \\\n--dataset_name \"smangrul/ultrachat-10k-chatml\" \\\n--chat_template_format \"chatml\" \\\n--add_special_tokens False \\\n--append_concat_token False \\\n--splits \"train,test\" \\\n--max_seq_len 2048 \\\n--num_train_epochs 1 \\\n--logging_steps 5 \\\n--log_level \"info\" \\\n--logging_strategy \"steps\" \\\n--evaluation_strategy \"epoch\" \\\n--save_strategy \"epoch\" \\\n--push_to_hub \\\n--hub_private_repo True \\\n--hub_strategy \"every_save\" \\\n--bf16 True \\\n--packing True \\\n--learning_rate 1e-4 \\\n--lr_scheduler_type \"cosine\" \\\n--weight_decay 1e-4 \\\n--warmup_ratio 0.0 \\\n--max_grad_norm 1.0 \\\n--output_dir \"llama-sft-qlora-dsz3\" \\\n--per_device_train_batch_size 2 \\\n--per_device_eval_batch_size 2 \\\n--gradient_accumulation_steps 2 \\\n--gradient_checkpointing True \\\n--use_reentrant True \\\n--dataset_text_field \"content\" \\\n--use_flash_attn True \\\n--use_peft_lora True \\\n--lora_r 8 \\\n--lora_alpha 16 \\\n--lora_dropout 0.1 \\\n--lora_target_modules \"all-linear\" \\\n--use_4bit_quantization True \\\n--use_nested_quant True \\\n--bnb_4bit_compute_dtype \"bfloat16\" \\\n--bnb_4bit_quant_storage_dtype \"bfloat16\"\n```\n\nNotice the new argument being passed `bnb_4bit_quant_storage_dtype` which denotes the data type for packing the 4-bit parameters."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 1a8bf6d1-86bf-46c7-af85-2a66b11f5e90
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"1a8bf6d1-86bf-46c7-af85-2a66b11f5e90\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6c70689e-ed8c-42ab-a308-9ee88c6ca160\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3af7059f34e85d08284b0ec018b7d50bbce4c55784a2f8a5ba0bc04d9e9be4ab\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d7b8cd93-7b7b-4013-965b-1b57f2b1b851\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2c0465f219113ebe965f2dd8c41d03164d29d5de9370a195390572a4eae1268d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11725, \"end_char_idx\": 14850, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "For example, when it is set to `bfloat16`, **32/4 = 8** 4-bit params are packed together post quantization.\n\nIn terms of training code, the important code changes are: \n\n```diff\n...\n\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=args.use_4bit_quantization,\n    bnb_4bit_quant_type=args.bnb_4bit_quant_type,\n    bnb_4bit_compute_dtype=compute_dtype,\n    bnb_4bit_use_double_quant=args.use_nested_quant,\n+   bnb_4bit_quant_storage=quant_storage_dtype,\n)\n\n...\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    args.model_name_or_path,\n    quantization_config=bnb_config,\n    trust_remote_code=True,\n    attn_implementation=\"flash_attention_2\" if args.use_flash_attn else \"eager\",\n+   torch_dtype=quant_storage_dtype or torch.float32,\n)\n```\n\nNotice that `torch_dtype` for `AutoModelForCausalLM` is same as the `bnb_4bit_quant_storage` data type. That's it. Everything else is handled by Trainer and TRL.\n\n## Memory usage\n\nIn the above example, the memory consumed per GPU is **36.6 GB**. Therefore, what took 8X80GB GPUs with DeepSpeed Stage 3+LoRA and a couple of 80GB GPUs with DDP+QLoRA now requires 2X40GB GPUs. This makes finetuning of large models more accessible.\n\n# Use PEFT and DeepSpeed with ZeRO3 and CPU Offloading for finetuning large models on a single GPU\nThis section of guide will help you learn how to use our DeepSpeed [training script](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py). You'll configure the script to train a large model for conditional generation with ZeRO-3 and CPU Offload.\n\n<Tip>\n\n\ud83d\udca1 To help you get started, check out our example training scripts for [causal language modeling](https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py) and [conditional generation](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py). You can adapt these scripts for your own applications or even use them out of the box if your task is similar to the one in the scripts.\n\n</Tip>\n\n## Configuration\n\nStart by running the following command to [create a DeepSpeed configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file ds_zero3_cpu.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll use ZeRO-3 along with CPU-Offload so make sure you pick those options.\n\n```bash\n`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d7b8cd93-7b7b-4013-965b-1b57f2b1b851
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d7b8cd93-7b7b-4013-965b-1b57f2b1b851\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1a8bf6d1-86bf-46c7-af85-2a66b11f5e90\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"12bef7d390b1547dae9eae1178de204561ee12ddaea73b61da3909017aede897\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"54f4459e-6e0f-4b85-9c0c-4050920cd2ae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f6330bc74bc55339c1f0f862843496ef5b85a291838e229ebb6862d275af21be\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13211, \"end_char_idx\": 15953, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "You'll configure the script to train a large model for conditional generation with ZeRO-3 and CPU Offload.\n\n<Tip>\n\n\ud83d\udca1 To help you get started, check out our example training scripts for [causal language modeling](https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py) and [conditional generation](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py). You can adapt these scripts for your own applications or even use them out of the box if your task is similar to the one in the scripts.\n\n</Tip>\n\n## Configuration\n\nStart by running the following command to [create a DeepSpeed configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file ds_zero3_cpu.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll use ZeRO-3 along with CPU-Offload so make sure you pick those options.\n\n```bash\n`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them.\n`gradient_clipping`: Enable gradient clipping with value.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO >= Stage-2.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3.\n`zero3_init_flag`: Decides whether to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with ZeRO Stage-3.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3.\n`mixed_precision`: `no` for FP32 training, `fp16` for FP16 mixed-precision training and `bf16` for BF16 mixed-precision training. \n```\n\nAn example [configuration file](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml) might look like the following. The most important thing to notice is that `zero_stage` is set to `3`, and `offload_optimizer_device` and `offload_param_device` are set to the `cpu`."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 54f4459e-6e0f-4b85-9c0c-4050920cd2ae
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"54f4459e-6e0f-4b85-9c0c-4050920cd2ae\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d7b8cd93-7b7b-4013-965b-1b57f2b1b851\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b7bc127db86909cbe51ffce506bb75c7444b9c708a88a64a0d17d110efce6b23\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7ac09438-55b9-44f7-b33e-52cb4d9bb555\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"730c2eb16a938c850f1c2e84cd5f258d0e4cf803536104dab55bda3f7c9fd1a7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14477, \"end_char_idx\": 17438, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "In this example, you'll use ZeRO-3 along with CPU-Offload so make sure you pick those options.\n\n```bash\n`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them.\n`gradient_clipping`: Enable gradient clipping with value.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO >= Stage-2.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3.\n`zero3_init_flag`: Decides whether to enable `deepspeed.zero.Init` for constructing massive models. Only applicable with ZeRO Stage-3.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3.\n`mixed_precision`: `no` for FP32 training, `fp16` for FP16 mixed-precision training and `bf16` for BF16 mixed-precision training. \n```\n\nAn example [configuration file](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml) might look like the following. The most important thing to notice is that `zero_stage` is set to `3`, and `offload_optimizer_device` and `offload_param_device` are set to the `cpu`.\n\n```yml\ncompute_environment: LOCAL_MACHINE\ndeepspeed_config:\n  gradient_accumulation_steps: 1\n  gradient_clipping: 1.0\n  offload_optimizer_device: cpu\n  offload_param_device: cpu\n  zero3_init_flag: true\n  zero3_save_16bit_model: true\n  zero_stage: 3\ndistributed_type: DEEPSPEED\ndowncast_bf16: 'no'\ndynamo_backend: 'NO'\nfsdp_config: {}\nmachine_rank: 0\nmain_training_function: main\nmegatron_lm_config: {}\nmixed_precision: 'no'\nnum_machines: 1\nnum_processes: 1\nrdzv_backend: static\nsame_network: true\nuse_cpu: false\n```\n\n## The important parts\n\nLet's dive a little deeper into the script so you can see what's going on, and understand how it works.\n\nWithin the [`main`](https://github.com/huggingface/peft/blob/2822398fbe896f25d4dac5e468624dc5fd65a51b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py#L103) function, the script creates an [`~accelerate.Accelerator`] class to initialize all the necessary requirements for distributed training.\n\n<Tip>\n\n\ud83d\udca1 Feel free to change the model and dataset inside the `main` function. If your dataset format is different from the one in the script, you may also need to write your own preprocessing function. \n\n</Tip>\n\nThe script also creates a configuration for the \ud83e\udd17 PEFT method you're using, which in this case, is LoRA. The [`LoraConfig`] specifies the task type and important parameters such as the dimension of the low-rank matrices, the matrices scaling factor, and the dropout probability of the LoRA layers."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 7ac09438-55b9-44f7-b33e-52cb4d9bb555
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"7ac09438-55b9-44f7-b33e-52cb4d9bb555\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"54f4459e-6e0f-4b85-9c0c-4050920cd2ae\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"1346aa36bba955fc7ed7f1cb1b20d8e1a2e6789a253b3904af584f588897aebd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52157146-c39b-446f-b94c-e6b78da71186\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"333b2d0480bf207e89fa8b7f0c1af185fd2978f43842c92fc9022bcc486ac57d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15955, \"end_char_idx\": 18828, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "```yml\ncompute_environment: LOCAL_MACHINE\ndeepspeed_config:\n  gradient_accumulation_steps: 1\n  gradient_clipping: 1.0\n  offload_optimizer_device: cpu\n  offload_param_device: cpu\n  zero3_init_flag: true\n  zero3_save_16bit_model: true\n  zero_stage: 3\ndistributed_type: DEEPSPEED\ndowncast_bf16: 'no'\ndynamo_backend: 'NO'\nfsdp_config: {}\nmachine_rank: 0\nmain_training_function: main\nmegatron_lm_config: {}\nmixed_precision: 'no'\nnum_machines: 1\nnum_processes: 1\nrdzv_backend: static\nsame_network: true\nuse_cpu: false\n```\n\n## The important parts\n\nLet's dive a little deeper into the script so you can see what's going on, and understand how it works.\n\nWithin the [`main`](https://github.com/huggingface/peft/blob/2822398fbe896f25d4dac5e468624dc5fd65a51b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py#L103) function, the script creates an [`~accelerate.Accelerator`] class to initialize all the necessary requirements for distributed training.\n\n<Tip>\n\n\ud83d\udca1 Feel free to change the model and dataset inside the `main` function. If your dataset format is different from the one in the script, you may also need to write your own preprocessing function. \n\n</Tip>\n\nThe script also creates a configuration for the \ud83e\udd17 PEFT method you're using, which in this case, is LoRA. The [`LoraConfig`] specifies the task type and important parameters such as the dimension of the low-rank matrices, the matrices scaling factor, and the dropout probability of the LoRA layers. If you want to use a different \ud83e\udd17 PEFT method, make sure you replace `LoraConfig` with the appropriate [class](../package_reference/tuners).\n\n```diff\n def main():\n+    accelerator = Accelerator()\n     model_name_or_path = \"facebook/bart-large\"\n     dataset_name = \"twitter_complaints\"\n+    peft_config = LoraConfig(\n         task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1\n     )\n```\n\nThroughout the script, you'll see the [`~accelerate.Accelerator.main_process_first`] and [`~accelerate.Accelerator.wait_for_everyone`] functions which help control and synchronize when processes are executed.\n\nThe [`get_peft_model`] function takes a base model and the [`peft_config`] you prepared earlier to create a [`PeftModel`]:\n\n```diff\n  model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n+ model = get_peft_model(model, peft_config)\n```\n\nPass all the relevant training objects to \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.prepare`] which makes sure everything is ready for training:\n\n```py\nmodel, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler = accelerator.prepare(\n    model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler\n)\n```\n\nThe next bit of code checks whether the DeepSpeed plugin is used in the `Accelerator`, and if the plugin exists, then we check if we are using ZeRO-3."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 52157146-c39b-446f-b94c-e6b78da71186
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"52157146-c39b-446f-b94c-e6b78da71186\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7ac09438-55b9-44f7-b33e-52cb4d9bb555\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d4dd8b9cfaeb99206930bec67a5284a6337ce7b320ec96f529ce966c339b9e0b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"55cd0c00-f47a-4fd8-b3b8-2f9d2f733b26\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ddbacb08848a001afbdaca34feff51d6ed7b926aa72728a0ab22536f19ffec13\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17246, \"end_char_idx\": 19976, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "The [`LoraConfig`] specifies the task type and important parameters such as the dimension of the low-rank matrices, the matrices scaling factor, and the dropout probability of the LoRA layers. If you want to use a different \ud83e\udd17 PEFT method, make sure you replace `LoraConfig` with the appropriate [class](../package_reference/tuners).\n\n```diff\n def main():\n+    accelerator = Accelerator()\n     model_name_or_path = \"facebook/bart-large\"\n     dataset_name = \"twitter_complaints\"\n+    peft_config = LoraConfig(\n         task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1\n     )\n```\n\nThroughout the script, you'll see the [`~accelerate.Accelerator.main_process_first`] and [`~accelerate.Accelerator.wait_for_everyone`] functions which help control and synchronize when processes are executed.\n\nThe [`get_peft_model`] function takes a base model and the [`peft_config`] you prepared earlier to create a [`PeftModel`]:\n\n```diff\n  model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n+ model = get_peft_model(model, peft_config)\n```\n\nPass all the relevant training objects to \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.prepare`] which makes sure everything is ready for training:\n\n```py\nmodel, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler = accelerator.prepare(\n    model, train_dataloader, eval_dataloader, test_dataloader, optimizer, lr_scheduler\n)\n```\n\nThe next bit of code checks whether the DeepSpeed plugin is used in the `Accelerator`, and if the plugin exists, then we check if we are using ZeRO-3. This conditional flag is used when calling `generate` function call during inference for syncing GPUs when the model parameters are sharded:\n\n```py\nis_ds_zero_3 = False\nif getattr(accelerator.state, \"deepspeed_plugin\", None):\n    is_ds_zero_3 = accelerator.state.deepspeed_plugin.zero_stage == 3\n```\n\nInside the training loop, the usual `loss.backward()` is replaced by \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.backward`] which uses the correct `backward()` method based on your configuration:\n\n```diff\n  for epoch in range(num_epochs):\n      with TorchTracemalloc() as tracemalloc:\n          model.train()\n          total_loss = 0\n          for step, batch in enumerate(tqdm(train_dataloader)):\n              outputs = model(**batch)\n              loss = outputs.loss\n              total_loss += loss.detach().float()\n+             accelerator.backward(loss)\n              optimizer.step()\n              lr_scheduler.step()\n              optimizer.zero_grad()\n```\n\nThat is all! The rest of the script handles the training loop, evaluation, and even pushes it to the Hub for you.\n\n## Train\n\nRun the following command to launch the training script."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 55cd0c00-f47a-4fd8-b3b8-2f9d2f733b26
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"55cd0c00-f47a-4fd8-b3b8-2f9d2f733b26\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"52157146-c39b-446f-b94c-e6b78da71186\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"78f5401e4f6de8ba06a71e1ef8e95e632b61b3a9efb458408e019028eeb580a5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e0b04d89-a596-4088-96c8-15c32974a9ae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"954d41d9139549f58ae1ed83463ac48ac0bb86ef8e0817ee758c8334aca8a97c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18829, \"end_char_idx\": 21931, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "This conditional flag is used when calling `generate` function call during inference for syncing GPUs when the model parameters are sharded:\n\n```py\nis_ds_zero_3 = False\nif getattr(accelerator.state, \"deepspeed_plugin\", None):\n    is_ds_zero_3 = accelerator.state.deepspeed_plugin.zero_stage == 3\n```\n\nInside the training loop, the usual `loss.backward()` is replaced by \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.backward`] which uses the correct `backward()` method based on your configuration:\n\n```diff\n  for epoch in range(num_epochs):\n      with TorchTracemalloc() as tracemalloc:\n          model.train()\n          total_loss = 0\n          for step, batch in enumerate(tqdm(train_dataloader)):\n              outputs = model(**batch)\n              loss = outputs.loss\n              total_loss += loss.detach().float()\n+             accelerator.backward(loss)\n              optimizer.step()\n              lr_scheduler.step()\n              optimizer.zero_grad()\n```\n\nThat is all! The rest of the script handles the training loop, evaluation, and even pushes it to the Hub for you.\n\n## Train\n\nRun the following command to launch the training script. Earlier, you saved the configuration file to `ds_zero3_cpu.yaml`, so you'll need to pass the path to the launcher with the `--config_file` argument like this:\n\n```bash\naccelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py\n```\n\nYou'll see some output logs that track memory usage during training, and once it's completed, the script returns the accuracy and compares the predictions to the labels:\n\n```bash\nGPU Memory before entering the train : 1916\nGPU Memory consumed at the end of the train (end-begin): 66\nGPU Peak Memory consumed during the train (max-begin): 7488\nGPU Total Peak Memory consumed during the train (max): 9404\nCPU Memory before entering the train : 19411\nCPU Memory consumed at the end of the train (end-begin): 0\nCPU Peak Memory consumed during the train (max-begin): 0\nCPU Total Peak Memory consumed during the train (max): 19411\nepoch=4: train_ppl=tensor(1.0705, device='cuda:0') train_epoch_loss=tensor(0.0681, device='cuda:0')\n100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 7/7 [00:27<00:00,  3.92s/it]\nGPU Memory before entering the eval : 1982\nGPU Memory consumed at the end of the eval (end-begin): -66\nGPU Peak Memory consumed during the eval (max-begin): 672\nGPU Total Peak Memory consumed during the eval (max): 2654\nCPU Memory before entering the eval : 19411\nCPU Memory consumed at the end of the eval (end-begin): 0\nCPU Peak Memory consumed during the eval (max-begin): 0\nCPU Total Peak Memory consumed during the eval (max): 19411\naccuracy=100.0\neval_preds[:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint']\ndataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint']\n```\n\n# Caveats\n1."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e0b04d89-a596-4088-96c8-15c32974a9ae
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/deepspeed", "title": "DeepSpeed", "tokens": 5684, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e0b04d89-a596-4088-96c8-15c32974a9ae\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d05013a5-7b26-4ef9-b830-26a4786d7753\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a8f3a08892343bb16b602d1d372b58e4b913a6acb9e638cfd34b438f3febffe8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"55cd0c00-f47a-4fd8-b3b8-2f9d2f733b26\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/deepspeed\", \"title\": \"DeepSpeed\", \"tokens\": 5684, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"16f2c1cc245bd4d9767cacba11ed1f03ce8386ba7cc3a0ad1b47f61c4eb6628c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21932, \"end_char_idx\": 22373, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "ref_doc_id": "d05013a5-7b26-4ef9-b830-26a4786d7753", "chroma:document": "Merging when using PEFT and DeepSpeed is currently unsupported and will raise error.\n2. When using CPU offloading, the major gains from using PEFT to shrink the optimizer states and gradients to that of the adapter weights would be realized on CPU RAM and there won't be savings with respect to GPU memory.\n3. DeepSpeed Stage 3 and qlora when used with CPU offloading leads to more GPU memory usage when compared to disabling CPU offloading."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d95fa38a-c275-45ec-97b4-01d1cca5c903
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d95fa38a-c275-45ec-97b4-01d1cca5c903\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b9b9cf55-ae1d-4e08-b08f-ff2e1e00209d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bb17d6578658b265b0a8a1bee48436ebc259a182983d03744f5e6424db30a693\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1852, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Fully Sharded Data Parallel\n\n[Fully sharded data parallel](https://pytorch.org/docs/stable/fsdp.html) (FSDP) is developed for distributed training of large pretrained models up to 1T parameters. FSDP achieves this by sharding the model parameters, gradients, and optimizer states across data parallel processes and it can also offload sharded model parameters to a CPU. The memory efficiency afforded by FSDP allows you to scale training to larger batch or model sizes.\n\nBoth of these features are supported in \ud83e\udd17 Accelerate, and you can use them with \ud83e\udd17 PEFT. \n\n# Use PEFT and FSDP\nThis section of guide will help you learn how to use our DeepSpeed [training script](https://github.com/huggingface/peft/blob/main/examples/sft/train.py) for performing SFT. You'll configure the script to do SFT (supervised fine-tuning) of Llama-70B model with LoRA and FSDP on 8xH100 80GB GPUs on a single machine. You can configure it to scale to multiple machines by changing the accelerate config.\n\n## Configuration\n\nStart by running the following command to [create a FSDP configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file fsdp_config.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll answer the questionnaire as shown in the image below."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: b9b9cf55-ae1d-4e08-b08f-ff2e1e00209d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"b9b9cf55-ae1d-4e08-b08f-ff2e1e00209d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d95fa38a-c275-45ec-97b4-01d1cca5c903\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"5d95bc72532cc471552246006875ed2fdfa823ef0aeac28046f394eba56d5475\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d06ee78c-a80c-429f-bc72-a6bb941b3615\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f68c55ae1bbf2d0abd713fe743e242a446f0c32b8f02ca09b48aae70a30dec30\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 166, \"end_char_idx\": 4303, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "-->\n\n# Fully Sharded Data Parallel\n\n[Fully sharded data parallel](https://pytorch.org/docs/stable/fsdp.html) (FSDP) is developed for distributed training of large pretrained models up to 1T parameters. FSDP achieves this by sharding the model parameters, gradients, and optimizer states across data parallel processes and it can also offload sharded model parameters to a CPU. The memory efficiency afforded by FSDP allows you to scale training to larger batch or model sizes.\n\nBoth of these features are supported in \ud83e\udd17 Accelerate, and you can use them with \ud83e\udd17 PEFT. \n\n# Use PEFT and FSDP\nThis section of guide will help you learn how to use our DeepSpeed [training script](https://github.com/huggingface/peft/blob/main/examples/sft/train.py) for performing SFT. You'll configure the script to do SFT (supervised fine-tuning) of Llama-70B model with LoRA and FSDP on 8xH100 80GB GPUs on a single machine. You can configure it to scale to multiple machines by changing the accelerate config.\n\n## Configuration\n\nStart by running the following command to [create a FSDP configuration file](https://huggingface.co/docs/accelerate/quicktour#launching-your-distributed-script) with \ud83e\udd17 Accelerate. The `--config_file` flag allows you to save the configuration file to a specific location, otherwise it is saved as a `default_config.yaml` file in the \ud83e\udd17 Accelerate cache.\n\nThe configuration file is used to set the default options when you launch the training script.\n\n```bash\naccelerate config --config_file fsdp_config.yaml\n```\n\nYou'll be asked a few questions about your setup, and configure the following arguments. In this example, you'll answer the questionnaire as shown in the image below.\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/fsdp-peft-config.png\"/>\n</div>\n<small>Creating Accelerate's config to use FSDP</small>\n\nOnce this is done, the corresponding config should look like below and you can find it in config folder at [fsdp_config.yaml](https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config.yaml):\n\n```yml\ncompute_environment: LOCAL_MACHINE\ndebug: false\ndistributed_type: FSDP\ndowncast_bf16: 'no'\nfsdp_config:\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_backward_prefetch: BACKWARD_PRE\n  fsdp_cpu_ram_efficient_loading: true\n  fsdp_forward_prefetch: false\n  fsdp_offload_params: false\n  fsdp_sharding_strategy: FULL_SHARD\n  fsdp_state_dict_type: SHARDED_STATE_DICT\n  fsdp_sync_module_states: true\n  fsdp_use_orig_params: false\nmachine_rank: 0\nmain_training_function: main\nmixed_precision: bf16\nnum_machines: 1\nnum_processes: 8\nrdzv_backend: static\nsame_network: true\ntpu_env: []\ntpu_use_cluster: false\ntpu_use_sudo: false\nuse_cpu: false\n```\n\n## Launch command\n\nThe launch command is available at [run_peft_fsdp.sh](https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_fsdp.sh) and it is also shown below:\n```bash\naccelerate launch --config_file \"configs/fsdp_config.yaml\"  train.py \\\n--seed 100 \\\n--model_name_or_path \"meta-llama/Llama-2-70b-hf\" \\\n--dataset_name \"smangrul/ultrachat-10k-chatml\" \\\n--chat_template_format \"chatml\" \\\n--add_special_tokens False \\\n--append_concat_token False \\\n--splits \"train,test\" \\\n--max_seq_len 2048 \\\n--num_train_epochs 1 \\\n--logging_steps 5 \\\n--log_level \"info\" \\\n--logging_strategy \"steps\" \\\n--evaluation_strategy \"epoch\" \\\n--save_strategy \"epoch\" \\\n--push_to_hub \\\n--hub_private_repo True \\\n--hub_strategy \"every_save\" \\\n--bf16 True \\\n--packing True \\\n--learning_rate 1e-4 \\\n--lr_scheduler_type \"cosine\" \\\n--weight_decay 1e-4 \\\n--warmup_ratio 0.0 \\\n--max_grad_norm 1.0 \\\n--output_dir \"llama-sft-lora-fsdp\" \\\n--per_device_train_batch_size 8 \\\n--per_device_eval_batch_size 8 \\\n--gradient_accumulation_steps 4 \\\n--gradient_checkpointing True \\\n--use_reentrant False \\\n--dataset_text_field \"content\" \\\n--use_flash_attn True \\\n--use_peft_lora True \\\n--lora_r 8 \\\n--lora_alpha 16 \\\n--lora_dropout 0.1 \\\n--lora_target_modules \"all-linear\" \\\n--use_4bit_quantization False\n```\n\nNotice that we are using LoRA with  rank=8, alpha=16 and targeting all linear layers."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d06ee78c-a80c-429f-bc72-a6bb941b3615
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d06ee78c-a80c-429f-bc72-a6bb941b3615\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b9b9cf55-ae1d-4e08-b08f-ff2e1e00209d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"37dddc47a5e41d7a60faf0694254323dea6172486df629c21b6d1707de2aaba7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"344a3c37-ebaa-49c1-800a-c9474c6b3b2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b970ca0032147316c61e405ff862758072b30b22c85f13ba71a40ee71c5634a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4304, \"end_char_idx\": 7064, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "We are passing the FSDP config file and finetuning the 70B Llama model on a subset of the [ultrachat dataset](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k).\n\n## The important parts\n\nLet's dive a little deeper into the script so you can see what's going on, and understand how it works.\n\nThe first thing to know is that the script uses FSDP for distributed training as the FSDP config has been passed. The `SFTTrainer` class handles all the heavy lifting of creating PEFT model using the peft config that is passed. After that when you call `trainer.train()`, Trainer internally uses \ud83e\udd17 Accelerate to prepare model, optimizer and trainer using the FSDP config to create FSDP wrapped model which is then trained. The main code snippet is below:\n\n```python\n# trainer\ntrainer = SFTTrainer(\n    model=model,\n    tokenizer=tokenizer,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n    peft_config=peft_config,\n    packing=data_args.packing,\n    dataset_kwargs={\n        \"append_concat_token\": data_args.append_concat_token,\n        \"add_special_tokens\": data_args.add_special_tokens,\n    },\n    dataset_text_field=data_args.dataset_text_field,\n    max_seq_length=data_args.max_seq_length,\n)\ntrainer.accelerator.print(f\"{trainer.model}\")\nif model_args.use_peft_lora:\n    # handle PEFT+FSDP case\n    trainer.model.print_trainable_parameters()\n    if getattr(trainer.accelerator.state, \"fsdp_plugin\", None):\n        from peft.utils.other import fsdp_auto_wrap_policy\n\n        fsdp_plugin = trainer.accelerator.state.fsdp_plugin\n        fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(trainer.model)\n\n# train\ncheckpoint = None\nif training_args.resume_from_checkpoint is not None:\n    checkpoint = training_args.resume_from_checkpoint\ntrainer.train(resume_from_checkpoint=checkpoint)\n\n# saving final model\nif trainer.is_fsdp_enabled:\n    trainer.accelerator.state.fsdp_plugin.set_state_dict_type(\"FULL_STATE_DICT\")\ntrainer.save_model()\n```\n\n\nHere, one main thing to note currently when using FSDP with PEFT is that `use_orig_params` needs to be `False` to realize GPU memory savings. Due to `use_orig_params=False`, the auto wrap policy for FSDP needs to change so that trainable and non-trainable parameters are wrapped separately. This is done by the code snippt below which uses the util function `fsdp_auto_wrap_policy` from PEFT:\n\n```\nif getattr(trainer.accelerator.state, \"fsdp_plugin\", None):\n    from peft.utils.other import fsdp_auto_wrap_policy\n\n    fsdp_plugin = trainer.accelerator.state.fsdp_plugin\n    fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(trainer.model)\n```\n\n## Memory usage\n\nIn the above example, the memory consumed per GPU is  72-80 GB (90-98%) as seen in the screenshot below."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 344a3c37-ebaa-49c1-800a-c9474c6b3b2c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"344a3c37-ebaa-49c1-800a-c9474c6b3b2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d06ee78c-a80c-429f-bc72-a6bb941b3615\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"0974b68e64515697838868161597a638ee2ffa3605321584584a6444759c1614\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"17801f65-3f74-469b-800c-f5b0d1d64903\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c23ca357ef2216764e91b73579ed062d257eedbf16a99cd37abcdf66c112ad64\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6296, \"end_char_idx\": 8497, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "Here, one main thing to note currently when using FSDP with PEFT is that `use_orig_params` needs to be `False` to realize GPU memory savings. Due to `use_orig_params=False`, the auto wrap policy for FSDP needs to change so that trainable and non-trainable parameters are wrapped separately. This is done by the code snippt below which uses the util function `fsdp_auto_wrap_policy` from PEFT:\n\n```\nif getattr(trainer.accelerator.state, \"fsdp_plugin\", None):\n    from peft.utils.other import fsdp_auto_wrap_policy\n\n    fsdp_plugin = trainer.accelerator.state.fsdp_plugin\n    fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(trainer.model)\n```\n\n## Memory usage\n\nIn the above example, the memory consumed per GPU is  72-80 GB (90-98%) as seen in the screenshot below. The slight increase in GPU memory at the end is when saving the model using `FULL_STATE_DICT` state dict type instead of the `SHARDED_STATE_DICT` so that the model has adapter weights that can be loaded normally with `from_pretrained` method during inference:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_fsdp_mem_usage.png\"/>\n</div>\n<small>GPU memory usage for the training run</small>\n\n# Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs\n\nIn this section, we will look at how to use QLoRA and FSDP for finetuning 70B llama model on 2X24GB GPUs. [Answer.AI](https://www.answer.ai/) in collaboration with bitsandbytes and Hugging Face \ud83e\udd17 open sourced code enabling the usage of FSDP+QLoRA and explained the whole process in their insightful blogpost [You can now train a 70b language model at home](https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html). This is now integrated in Hugging Face ecosystem. \n\nFor this, we first need `bitsandbytes>=0.43.0`, `accelerate>=0.28.0`, `transformers>4.38.2`, `trl>0.7.11` and `peft>0.9.0`. We need to set `fsdp_cpu_ram_efficient_loading=true`, `fsdp_use_orig_params=false` and `fsdp_offload_params=true`(cpu offloading) when using Accelerate config. When not using accelerate launcher, you can alternately set the environment variable `export FSDP_CPU_RAM_EFFICIENT_LOADING=true`."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 17801f65-3f74-469b-800c-f5b0d1d64903
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"17801f65-3f74-469b-800c-f5b0d1d64903\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"344a3c37-ebaa-49c1-800a-c9474c6b3b2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"76a6a52b696610e53a5e88ba616d38c934ab3559b2a6c9e5319b39b6b0ebd64f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6df82749-b04b-484e-b084-7d05e4404652\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e076b03ce0e4933675f199d10e14388b2ad84892cbad3c37a0ac77d73b5ab138\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7065, \"end_char_idx\": 11159, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "The slight increase in GPU memory at the end is when saving the model using `FULL_STATE_DICT` state dict type instead of the `SHARDED_STATE_DICT` so that the model has adapter weights that can be loaded normally with `from_pretrained` method during inference:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_fsdp_mem_usage.png\"/>\n</div>\n<small>GPU memory usage for the training run</small>\n\n# Use PEFT QLoRA and FSDP for finetuning large models on multiple GPUs\n\nIn this section, we will look at how to use QLoRA and FSDP for finetuning 70B llama model on 2X24GB GPUs. [Answer.AI](https://www.answer.ai/) in collaboration with bitsandbytes and Hugging Face \ud83e\udd17 open sourced code enabling the usage of FSDP+QLoRA and explained the whole process in their insightful blogpost [You can now train a 70b language model at home](https://www.answer.ai/posts/2024-03-06-fsdp-qlora.html). This is now integrated in Hugging Face ecosystem. \n\nFor this, we first need `bitsandbytes>=0.43.0`, `accelerate>=0.28.0`, `transformers>4.38.2`, `trl>0.7.11` and `peft>0.9.0`. We need to set `fsdp_cpu_ram_efficient_loading=true`, `fsdp_use_orig_params=false` and `fsdp_offload_params=true`(cpu offloading) when using Accelerate config. When not using accelerate launcher, you can alternately set the environment variable `export FSDP_CPU_RAM_EFFICIENT_LOADING=true`.  Here, we will be using accelerate config and below is the config which can be found at [fsdp_config_qlora.yaml](https://github.com/huggingface/peft/blob/main/examples/sft/configs/fsdp_config_qlora.yaml):\n\n```yml\ncompute_environment: LOCAL_MACHINE                                                                                                                                           \ndebug: false                                                                                                                                                                 \ndistributed_type: FSDP\ndowncast_bf16: 'no'\nfsdp_config:\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_backward_prefetch: BACKWARD_PRE\n  fsdp_cpu_ram_efficient_loading: true\n  fsdp_forward_prefetch: false\n  fsdp_offload_params: true\n  fsdp_sharding_strategy: FULL_SHARD\n  fsdp_state_dict_type: SHARDED_STATE_DICT\n  fsdp_sync_module_states: true\n  fsdp_use_orig_params: false\nmachine_rank: 0\nmain_training_function: main\nmixed_precision: 'no'\nnum_machines: 1\nnum_processes: 2\nrdzv_backend: static\nsame_network: true\ntpu_env: []\ntpu_use_cluster: false\ntpu_use_sudo: false\nuse_cpu: false\n```\n\nLaunch command is given below which is available at [run_peft_qlora_fsdp.sh](https://github.com/huggingface/peft/blob/main/examples/sft/run_peft_qlora_fsdp.sh):\n```\naccelerate launch --config_file \"configs/fsdp_config_qlora.yaml\"  train.py \\\n--seed 100 \\\n--model_name_or_path \"meta-llama/Llama-2-70b-hf\" \\\n--dataset_name \"smangrul/ultrachat-10k-chatml\" \\\n--chat_template_format \"chatml\" \\\n--add_special_tokens False \\\n--append_concat_token False \\\n--splits \"train,test\" \\\n--max_seq_len 2048 \\\n--num_train_epochs 1 \\\n--logging_steps 5 \\\n--log_level \"info\" \\\n--logging_strategy \"steps\" \\\n--evaluation_strategy \"epoch\" \\\n--save_strategy \"epoch\" \\\n--push_to_hub \\\n--hub_private_repo True \\\n--hub_strategy \"every_save\" \\\n--bf16 True \\\n--packing True \\\n--learning_rate 1e-4 \\\n--lr_scheduler_type \"cosine\" \\\n--weight_decay 1e-4 \\\n--warmup_ratio 0.0 \\\n--max_grad_norm 1.0 \\\n--output_dir \"llama-sft-qlora-fsdp\" \\\n--per_device_train_batch_size 2 \\\n--per_device_eval_batch_size 2 \\\n--gradient_accumulation_steps 2 \\\n--gradient_checkpointing True \\\n--use_reentrant True \\\n--dataset_text_field \"content\" \\\n--use_flash_attn True \\\n--use_peft_lora True \\\n--lora_r 8 \\\n--lora_alpha 16 \\\n--lora_dropout 0.1 \\\n--lora_target_modules \"all-linear\" \\\n--use_4bit_quantization True \\\n--use_nested_quant True \\\n--bnb_4bit_compute_dtype \"bfloat16\" \\\n--bnb_4bit_quant_storage_dtype \"bfloat16\"\n```\n\nNotice the new argument being passed, `bnb_4bit_quant_storage_dtype`, which denotes the data type for packing the 4-bit parameters."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 6df82749-b04b-484e-b084-7d05e4404652
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/accelerate/fsdp", "title": "Fully Sharded Data Parallel", "tokens": 3543, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"6df82749-b04b-484e-b084-7d05e4404652\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0afd20a3-8fa5-44d1-8131-2e6e7d175826\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f24545eac4680c6e541d1631f4209b1d76c48e7e1c0c985a3760d14bbf1cbe46\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"17801f65-3f74-469b-800c-f5b0d1d64903\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/accelerate/fsdp\", \"title\": \"Fully Sharded Data Parallel\", \"tokens\": 3543, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"bf968998bf944d85afb9f716d0a9e62ce7b3f995eaf6a3e8608bd6c380b639db\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11160, \"end_char_idx\": 13671, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "ref_doc_id": "0afd20a3-8fa5-44d1-8131-2e6e7d175826", "chroma:document": "For example, when it is set to `bfloat16`, **16/4 = 4** 4-bit params are packed together post quantization. When using mixed precision training with `bfloat16`, `bnb_4bit_quant_storage_dtype` can be either `bfloat16` for pure `bfloat16` finetuning, or `float32` for automatic mixed precision (this consumes more GPU memory). When using mixed precision training with `float16`, `bnb_4bit_quant_storage_dtype` should be set to `float32` for stable automatic mixed precision training.\n\nIn terms of training code, the important code changes are: \n\n```diff\n...\n\nbnb_config = BitsAndBytesConfig(\n    load_in_4bit=args.use_4bit_quantization,\n    bnb_4bit_quant_type=args.bnb_4bit_quant_type,\n    bnb_4bit_compute_dtype=compute_dtype,\n    bnb_4bit_use_double_quant=args.use_nested_quant,\n+   bnb_4bit_quant_storage=quant_storage_dtype,\n)\n\n...\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    args.model_name_or_path,\n    quantization_config=bnb_config,\n    trust_remote_code=True,\n    attn_implementation=\"flash_attention_2\" if args.use_flash_attn else \"eager\",\n+   torch_dtype=quant_storage_dtype or torch.float32,\n)\n```\n\nNotice that `torch_dtype` for `AutoModelForCausalLM` is same as the `bnb_4bit_quant_storage` data type. That's it. Everything else is handled by Trainer and TRL.\n\n## Memory usage\n\nIn the above example, the memory consumed per GPU is **19.6 GB** while CPU RAM usage is around **107 GB**. When disabling CPU offloading, the GPU memory usage is  **35.6 GB/ GPU**. Therefore, what took 16X80GB GPUs for full finetuning, 8X80GB GPUs with FSDP+LoRA, and a couple of 80GB GPUs with DDP+QLoRA, now requires 2X24GB GPUs. This makes finetuning of large models more accessible.\n\n## More resources\nYou can also refer the [llama-recipes](https://github.com/facebookresearch/llama-recipes/?tab=readme-ov-file#fine-tuning) repo and [Getting started with Llama](https://llama.meta.com/get-started/#fine-tuning) guide on how to finetune using FSDP and PEFT.\n\n## Caveats\n1. Merging when using PEFT and FSDP is currently unsupported and will raise error.\n2. Passing `modules_to_save` config parameter to is untested at present.\n3. GPU Memory saving when using CPU Offloading is untested at present.\n4. When using FSDP+QLoRA, `paged_adamw_8bit` currently results in an error when saving a checkpoint.\n5. DoRA training with FSDP should work (albeit at lower speed than LoRA). If combined with bitsandbytes (QDoRA), 4-bit quantization should also work, but 8-bit quantization has known issues and is not recommended."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: f81e2176-68f5-44a0-aefe-5593be9740e2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_integrations", "title": "PEFT integrations", "tokens": 1791, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"f81e2176-68f5-44a0-aefe-5593be9740e2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d4dee759-53ee-40ec-99d1-8a9cd5ab00cb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"beee27f402e7f8e884ebb561a6e03714e8a0baab4c163b7e6c4b3756d4860328\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8a06ae17-2a00-4b62-9960-d053c492d6a7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5879ce00922e12702e5d40196aa0b57c85504d5c8fe23ac53848b70dac520b27\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2588, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "ref_doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "chroma:document": "# PEFT integrations\n\nPEFT's practical benefits extends to other Hugging Face libraries like [Diffusers](https://hf.co/docs/diffusers) and [Transformers](https://hf.co/docs/transformers). One of the main benefits of PEFT is that an adapter file generated by a PEFT method is a lot smaller than the original model, which makes it super easy to manage and use multiple adapters. You can use one pretrained base model for multiple tasks by simply loading a new adapter finetuned for the task you're solving. Or you can combine multiple adapters with a text-to-image diffusion model to create new effects.\n\nThis tutorial will show you how PEFT can help you manage adapters in Diffusers and Transformers.\n\n## Diffusers\n\nDiffusers is a generative AI library for creating images and videos from text or images with diffusion models. LoRA is an especially popular training method for diffusion models because you can very quickly train and share diffusion models to generate images in new styles. To make it easier to use and try multiple LoRA models, Diffusers uses the PEFT library to help manage different adapters for inference.\n\nFor example, load a base model and then load the [artificialguybr/3DRedmond-V1](https://huggingface.co/artificialguybr/3DRedmond-V1) adapter for inference with the [`load_lora_weights`](https://huggingface.co/docs/diffusers/v0.24.0/en/api/loaders/lora#diffusers.loaders.LoraLoaderMixin.load_lora_weights) method. The `adapter_name` argument in the loading method is enabled by PEFT and allows you to set a name for the adapter so it is easier to reference.\n\n```py\nimport torch\nfrom diffusers import DiffusionPipeline\n\npipeline = DiffusionPipeline.from_pretrained(\n    \"stabilityai/stable-diffusion-xl-base-1.0\", torch_dtype=torch.float16\n).to(\"cuda\")\npipeline.load_lora_weights(\n    \"peft-internal-testing/artificialguybr__3DRedmond-V1\", \n    weight_name=\"3DRedmond-3DRenderStyle-3DRenderAF.safetensors\", \n    adapter_name=\"3d\"\n)\nimage = pipeline(\"sushi rolls shaped like kawaii cat faces\").images[0]\nimage\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/test-lora-diffusers.png\"/>\n</div>\n\nNow let's try another cool LoRA model, [ostris/super-cereal-sdxl-lora](https://huggingface.co/ostris/super-cereal-sdxl-lora). All you need to do is load and name this new adapter with `adapter_name`, and use the [`set_adapters`](https://huggingface.co/docs/diffusers/api/loaders/unet#diffusers.loaders.UNet2DConditionLoadersMixin.set_adapters) method to set it as the currently active adapter."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 8a06ae17-2a00-4b62-9960-d053c492d6a7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_integrations", "title": "PEFT integrations", "tokens": 1791, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"8a06ae17-2a00-4b62-9960-d053c492d6a7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d4dee759-53ee-40ec-99d1-8a9cd5ab00cb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"beee27f402e7f8e884ebb561a6e03714e8a0baab4c163b7e6c4b3756d4860328\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f81e2176-68f5-44a0-aefe-5593be9740e2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"66f34336f5392a473d5a36a286f0c143dcbaaededcc5fe9b089f9029ad34941c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"38e9e9fa-be34-4968-8062-e8fe17ec554c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0a64614eb2ccf66d3526343f47936963cb04dacff61b0b8980cb94569d986f18\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1438, \"end_char_idx\": 4387, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "ref_doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "chroma:document": "The `adapter_name` argument in the loading method is enabled by PEFT and allows you to set a name for the adapter so it is easier to reference.\n\n```py\nimport torch\nfrom diffusers import DiffusionPipeline\n\npipeline = DiffusionPipeline.from_pretrained(\n    \"stabilityai/stable-diffusion-xl-base-1.0\", torch_dtype=torch.float16\n).to(\"cuda\")\npipeline.load_lora_weights(\n    \"peft-internal-testing/artificialguybr__3DRedmond-V1\", \n    weight_name=\"3DRedmond-3DRenderStyle-3DRenderAF.safetensors\", \n    adapter_name=\"3d\"\n)\nimage = pipeline(\"sushi rolls shaped like kawaii cat faces\").images[0]\nimage\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/test-lora-diffusers.png\"/>\n</div>\n\nNow let's try another cool LoRA model, [ostris/super-cereal-sdxl-lora](https://huggingface.co/ostris/super-cereal-sdxl-lora). All you need to do is load and name this new adapter with `adapter_name`, and use the [`set_adapters`](https://huggingface.co/docs/diffusers/api/loaders/unet#diffusers.loaders.UNet2DConditionLoadersMixin.set_adapters) method to set it as the currently active adapter.\n\n```py\npipeline.load_lora_weights(\n    \"ostris/super-cereal-sdxl-lora\", \n    weight_name=\"cereal_box_sdxl_v1.safetensors\", \n    adapter_name=\"cereal\"\n)\npipeline.set_adapters(\"cereal\")\nimage = pipeline(\"sushi rolls shaped like kawaii cat faces\").images[0]\nimage\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/test-lora-diffusers-2.png\"/>\n</div>\n\nFinally, you can call the [`disable_lora`](https://huggingface.co/docs/diffusers/api/loaders/unet#diffusers.loaders.UNet2DConditionLoadersMixin.disable_lora) method to restore the base model.\n\n```py\npipeline.disable_lora()\n```\n\nLearn more about how PEFT supports Diffusers in the [Inference with PEFT](https://huggingface.co/docs/diffusers/tutorials/using_peft_for_inference) tutorial.\n\n## Transformers\n\n\ud83e\udd17 [Transformers](https://hf.co/docs/transformers) is a collection of pretrained models for all types of tasks in all modalities. You can load these models for training or inference. Many of the models are large language models (LLMs), so it makes sense to integrate PEFT with Transformers to manage and train adapters.\n\nLoad a base pretrained model to train.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n```\n\nNext, add an adapter configuration to specify how to adapt the model parameters. Call the [`~PeftModel.add_adapter`] method to add the configuration to the base model.\n\n```py\nfrom peft import LoraConfig\n\npeft_config = LoraConfig(\n    lora_alpha=16,\n    lora_dropout=0.1,\n    r=64,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\"\n)\nmodel.add_adapter(peft_config)\n```\n\nNow you can train the model with Transformer's [`~transformers.Trainer`] class or whichever training framework you prefer."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 38e9e9fa-be34-4968-8062-e8fe17ec554c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_integrations", "title": "PEFT integrations", "tokens": 1791, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"38e9e9fa-be34-4968-8062-e8fe17ec554c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d4dee759-53ee-40ec-99d1-8a9cd5ab00cb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"beee27f402e7f8e884ebb561a6e03714e8a0baab4c163b7e6c4b3756d4860328\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8a06ae17-2a00-4b62-9960-d053c492d6a7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f2987d875c9e1d26ed7684beb437c0472c93df5892125fd809edb239ae2ab68a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"bceedfa6-dba0-4006-8cbf-f07772881f83\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cfd468203abad5001477ccee51bab74472fb516e9cd4d90d4c8650b0fb64f6a6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3208, \"end_char_idx\": 6099, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "ref_doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "chroma:document": "```py\npipeline.disable_lora()\n```\n\nLearn more about how PEFT supports Diffusers in the [Inference with PEFT](https://huggingface.co/docs/diffusers/tutorials/using_peft_for_inference) tutorial.\n\n## Transformers\n\n\ud83e\udd17 [Transformers](https://hf.co/docs/transformers) is a collection of pretrained models for all types of tasks in all modalities. You can load these models for training or inference. Many of the models are large language models (LLMs), so it makes sense to integrate PEFT with Transformers to manage and train adapters.\n\nLoad a base pretrained model to train.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n```\n\nNext, add an adapter configuration to specify how to adapt the model parameters. Call the [`~PeftModel.add_adapter`] method to add the configuration to the base model.\n\n```py\nfrom peft import LoraConfig\n\npeft_config = LoraConfig(\n    lora_alpha=16,\n    lora_dropout=0.1,\n    r=64,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\"\n)\nmodel.add_adapter(peft_config)\n```\n\nNow you can train the model with Transformer's [`~transformers.Trainer`] class or whichever training framework you prefer.\n\nTo use the newly trained model for inference, the [`~transformers.AutoModel`] class uses PEFT on the backend to load the adapter weights and configuration file into a base pretrained model.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"peft-internal-testing/opt-350m-lora\")\n```\n\nAlternatively, you can use transformers [Pipelines](https://huggingface.co/docs/transformers/en/main_classes/pipelines) to load the model for conveniently running inference:\n\n```py\nfrom transformers import pipeline\n\nmodel = pipeline(\"text-generation\", \"peft-internal-testing/opt-350m-lora\")\nprint(model(\"Hello World\"))\n```\n\nIf you're interested in comparing or using more than one adapter, you can call the [`~PeftModel.add_adapter`] method to add the adapter configuration to the base model. The only requirement is the adapter type must be the same (you can't mix a LoRA and LoHa adapter).\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import LoraConfig\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\nmodel.add_adapter(lora_config_1, adapter_name=\"adapter_1\")\n```\n\nCall [`~PeftModel.add_adapter`] again to attach a new adapter to the base model.\n\n```py\nmodel.add_adapter(lora_config_2, adapter_name=\"adapter_2\")\n```\n\nThen you can use [`~PeftModel.set_adapter`] to set the currently active adapter.\n\n```py\nmodel.set_adapter(\"adapter_1\")\noutput = model.generate(**inputs)\nprint(tokenizer.decode(output_disabled[0], skip_special_tokens=True))\n```\n\nTo disable the adapter, call the [disable_adapters](https://github.com/huggingface/transformers/blob/4e3490f79b40248c53ee54365a9662611e880892/src/transformers/integrations/peft.py#L313) method."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: bceedfa6-dba0-4006-8cbf-f07772881f83
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_integrations", "title": "PEFT integrations", "tokens": 1791, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"bceedfa6-dba0-4006-8cbf-f07772881f83\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d4dee759-53ee-40ec-99d1-8a9cd5ab00cb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"beee27f402e7f8e884ebb561a6e03714e8a0baab4c163b7e6c4b3756d4860328\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"38e9e9fa-be34-4968-8062-e8fe17ec554c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_integrations\", \"title\": \"PEFT integrations\", \"tokens\": 1791, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"47d2b9d1f6b405839931f407cd7c0000863228ba0043bdd257995b73f1b19c54\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4580, \"end_char_idx\": 6480, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "ref_doc_id": "d4dee759-53ee-40ec-99d1-8a9cd5ab00cb", "chroma:document": "```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"peft-internal-testing/opt-350m-lora\")\n```\n\nAlternatively, you can use transformers [Pipelines](https://huggingface.co/docs/transformers/en/main_classes/pipelines) to load the model for conveniently running inference:\n\n```py\nfrom transformers import pipeline\n\nmodel = pipeline(\"text-generation\", \"peft-internal-testing/opt-350m-lora\")\nprint(model(\"Hello World\"))\n```\n\nIf you're interested in comparing or using more than one adapter, you can call the [`~PeftModel.add_adapter`] method to add the adapter configuration to the base model. The only requirement is the adapter type must be the same (you can't mix a LoRA and LoHa adapter).\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import LoraConfig\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\nmodel.add_adapter(lora_config_1, adapter_name=\"adapter_1\")\n```\n\nCall [`~PeftModel.add_adapter`] again to attach a new adapter to the base model.\n\n```py\nmodel.add_adapter(lora_config_2, adapter_name=\"adapter_2\")\n```\n\nThen you can use [`~PeftModel.set_adapter`] to set the currently active adapter.\n\n```py\nmodel.set_adapter(\"adapter_1\")\noutput = model.generate(**inputs)\nprint(tokenizer.decode(output_disabled[0], skip_special_tokens=True))\n```\n\nTo disable the adapter, call the [disable_adapters](https://github.com/huggingface/transformers/blob/4e3490f79b40248c53ee54365a9662611e880892/src/transformers/integrations/peft.py#L313) method.\n\n```py\nmodel.disable_adapters()\n```\n\nThe [enable_adapters](https://github.com/huggingface/transformers/blob/4e3490f79b40248c53ee54365a9662611e880892/src/transformers/integrations/peft.py#L336) can be used to enable the adapters again.\n\nIf you're curious, check out the [Load and train adapters with PEFT](https://huggingface.co/docs/transformers/main/peft) tutorial to learn more."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d0ea62ff-1de6-40e1-b498-393d2d03e344
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_model_config", "title": "PEFT configurations and models", "tokens": 1940, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d0ea62ff-1de6-40e1-b498-393d2d03e344\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"5ac96aa4-9551-4087-b933-4d09cbf3fd9b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"36f7eede49bcbb1fbca50686517a245cfc6d4d7d3a86c2d0d5cb453d0dbc1f4a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"133894fc-55ea-4fd2-b470-af509019e3ba\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"15f881a3a252100013d50c57e82c58285fd4f1a90e5b98e59fd06d971f83e975\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2656, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "ref_doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "chroma:document": "# PEFT configurations and models\n\nThe sheer size of today's large pretrained models - which commonly have billions of parameters - present a significant training challenge because they require more storage space and more computational power to crunch all those calculations. You'll need access to powerful GPUs or TPUs to train these large pretrained models which is expensive, not widely accessible to everyone, not environmentally friendly, and not very practical. PEFT methods address many of these challenges. There are several types of PEFT methods (soft prompting, matrix decomposition, adapters), but they all focus on the same thing, reduce the number of trainable parameters. This makes it more accessible to train and store large models on consumer hardware.\n\nThe PEFT library is designed to help you quickly train large models on free or low-cost GPUs, and in this tutorial, you'll learn how to setup a configuration to apply a PEFT method to a pretrained base model for training. Once the PEFT configuration is setup, you can use any training framework you like (Transformer's [`~transformers.Trainer`] class, [Accelerate](https://hf.co/docs/accelerate), a custom PyTorch training loop).\n\n## PEFT configurations\n\n<Tip>\n\nLearn more about the parameters you can configure for each PEFT method in their respective API reference page.\n\n</Tip>\n\nA configuration stores important parameters that specify how a particular PEFT method should be applied.\n\nFor example, take a look at the following [`LoraConfig`](https://huggingface.co/ybelkada/opt-350m-lora/blob/main/adapter_config.json) for applying LoRA and [`PromptEncoderConfig`](https://huggingface.co/smangrul/roberta-large-peft-p-tuning/blob/main/adapter_config.json) for applying p-tuning (these configuration files are already JSON-serialized). Whenever you load a PEFT adapter, it is a good idea to check whether it has an associated adapter_config.json file which is required.\n\n<hfoptions id=\"config\">\n<hfoption id=\"LoraConfig\">\n\n```json\n{\n  \"base_model_name_or_path\": \"facebook/opt-350m\", #base model to apply LoRA to\n  \"bias\": \"none\",\n  \"fan_in_fan_out\": false,\n  \"inference_mode\": true,\n  \"init_lora_weights\": true,\n  \"layers_pattern\": null,\n  \"layers_to_transform\": null,\n  \"lora_alpha\": 32,\n  \"lora_dropout\": 0.05,\n  \"modules_to_save\": null,\n  \"peft_type\": \"LORA\", #PEFT method type\n  \"r\": 16,\n  \"revision\": null,\n  \"target_modules\": [\n    \"q_proj\", #model modules to apply LoRA to (query and value projection layers)\n    \"v_proj\"\n  ],\n  \"task_type\": \"CAUSAL_LM\" #type of task to train model on\n}\n```\n\nYou can create your own configuration for training by initializing a [`LoraConfig`]."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 133894fc-55ea-4fd2-b470-af509019e3ba
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_model_config", "title": "PEFT configurations and models", "tokens": 1940, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"133894fc-55ea-4fd2-b470-af509019e3ba\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"5ac96aa4-9551-4087-b933-4d09cbf3fd9b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"36f7eede49bcbb1fbca50686517a245cfc6d4d7d3a86c2d0d5cb453d0dbc1f4a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d0ea62ff-1de6-40e1-b498-393d2d03e344\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"4fd119e835cf269dfeffca87417e8f93a31f34ac22075e206e21424360a3d30c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8987281f-4cf5-4e7c-bf8b-50c6243d967a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9384209e7d7115ffbbe481ba46ef191c0f6008d7f83f90fa1b971f5491311589\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1201, \"end_char_idx\": 3488, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "ref_doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "chroma:document": "## PEFT configurations\n\n<Tip>\n\nLearn more about the parameters you can configure for each PEFT method in their respective API reference page.\n\n</Tip>\n\nA configuration stores important parameters that specify how a particular PEFT method should be applied.\n\nFor example, take a look at the following [`LoraConfig`](https://huggingface.co/ybelkada/opt-350m-lora/blob/main/adapter_config.json) for applying LoRA and [`PromptEncoderConfig`](https://huggingface.co/smangrul/roberta-large-peft-p-tuning/blob/main/adapter_config.json) for applying p-tuning (these configuration files are already JSON-serialized). Whenever you load a PEFT adapter, it is a good idea to check whether it has an associated adapter_config.json file which is required.\n\n<hfoptions id=\"config\">\n<hfoption id=\"LoraConfig\">\n\n```json\n{\n  \"base_model_name_or_path\": \"facebook/opt-350m\", #base model to apply LoRA to\n  \"bias\": \"none\",\n  \"fan_in_fan_out\": false,\n  \"inference_mode\": true,\n  \"init_lora_weights\": true,\n  \"layers_pattern\": null,\n  \"layers_to_transform\": null,\n  \"lora_alpha\": 32,\n  \"lora_dropout\": 0.05,\n  \"modules_to_save\": null,\n  \"peft_type\": \"LORA\", #PEFT method type\n  \"r\": 16,\n  \"revision\": null,\n  \"target_modules\": [\n    \"q_proj\", #model modules to apply LoRA to (query and value projection layers)\n    \"v_proj\"\n  ],\n  \"task_type\": \"CAUSAL_LM\" #type of task to train model on\n}\n```\n\nYou can create your own configuration for training by initializing a [`LoraConfig`].\n\n```py\nfrom peft import LoraConfig, TaskType\n\nlora_config = LoraConfig(\n    r=16,\n    target_modules=[\"q_proj\", \"v_proj\"],\n    task_type=TaskType.CAUSAL_LM,\n    lora_alpha=32,\n    lora_dropout=0.05\n)\n```\n\n</hfoption>\n<hfoption id=\"PromptEncoderConfig\">\n\n```json\n{\n  \"base_model_name_or_path\": \"roberta-large\", #base model to apply p-tuning to\n  \"encoder_dropout\": 0.0,\n  \"encoder_hidden_size\": 128,\n  \"encoder_num_layers\": 2,\n  \"encoder_reparameterization_type\": \"MLP\",\n  \"inference_mode\": true,\n  \"num_attention_heads\": 16,\n  \"num_layers\": 24,\n  \"num_transformer_submodules\": 1,\n  \"num_virtual_tokens\": 20,\n  \"peft_type\": \"P_TUNING\", #PEFT method type\n  \"task_type\": \"SEQ_CLS\", #type of task to train model on\n  \"token_dim\": 1024\n}\n```\n\nYou can create your own configuration for training by initializing a [`PromptEncoderConfig`]."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 8987281f-4cf5-4e7c-bf8b-50c6243d967a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_model_config", "title": "PEFT configurations and models", "tokens": 1940, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"8987281f-4cf5-4e7c-bf8b-50c6243d967a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"5ac96aa4-9551-4087-b933-4d09cbf3fd9b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"36f7eede49bcbb1fbca50686517a245cfc6d4d7d3a86c2d0d5cb453d0dbc1f4a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"133894fc-55ea-4fd2-b470-af509019e3ba\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d1bff5f4b9f302033b47c83de79a0f7201a895a59eef695d62a14d40b86aaca0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52ef35f5-5b16-4a70-8784-f0035edabc3a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0dca32dbc6b30a8d4750dd4fc6c707c48b55494552a6d8d9eeaa7c6f2dd343d2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2658, \"end_char_idx\": 5290, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "ref_doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "chroma:document": "```py\nfrom peft import LoraConfig, TaskType\n\nlora_config = LoraConfig(\n    r=16,\n    target_modules=[\"q_proj\", \"v_proj\"],\n    task_type=TaskType.CAUSAL_LM,\n    lora_alpha=32,\n    lora_dropout=0.05\n)\n```\n\n</hfoption>\n<hfoption id=\"PromptEncoderConfig\">\n\n```json\n{\n  \"base_model_name_or_path\": \"roberta-large\", #base model to apply p-tuning to\n  \"encoder_dropout\": 0.0,\n  \"encoder_hidden_size\": 128,\n  \"encoder_num_layers\": 2,\n  \"encoder_reparameterization_type\": \"MLP\",\n  \"inference_mode\": true,\n  \"num_attention_heads\": 16,\n  \"num_layers\": 24,\n  \"num_transformer_submodules\": 1,\n  \"num_virtual_tokens\": 20,\n  \"peft_type\": \"P_TUNING\", #PEFT method type\n  \"task_type\": \"SEQ_CLS\", #type of task to train model on\n  \"token_dim\": 1024\n}\n```\n\nYou can create your own configuration for training by initializing a [`PromptEncoderConfig`].\n\n```py\nfrom peft import PromptEncoderConfig, TaskType\n\np_tuning_config = PromptEncoderConfig(\n    encoder_reparameterization_type=\"MLP\",\n    encoder_hidden_size=128,\n    num_attention_heads=16,\n    num_layers=24,\n    num_transformer_submodules=1,\n    num_virtual_tokens=20,\n    token_dim=1024,\n    task_type=TaskType.SEQ_CLS\n)\n```\n\n</hfoption>\n</hfoptions>\n\n## PEFT models\n\nWith a PEFT configuration in hand, you can now apply it to any pretrained model to create a [`PeftModel`]. Choose from any of the state-of-the-art models from the [Transformers](https://hf.co/docs/transformers) library, a custom model, and even new and unsupported transformer architectures.\n\nFor this tutorial, load a base [facebook/opt-350m](https://huggingface.co/facebook/opt-350m) model to finetune.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n```\n\nUse the [`get_peft_model`] function to create a [`PeftModel`] from the base facebook/opt-350m model and the `lora_config` you created earlier.\n\n```py\nfrom peft import get_peft_model\n\nlora_model = get_peft_model(model, lora_config)\nlora_model.print_trainable_parameters()\n\"trainable params: 1,572,864 || all params: 332,769,280 || trainable%: 0.472659014678278\"\n```\n\nNow you can train the [`PeftModel`] with your preferred training framework! After training, you can save your model locally with [`~PeftModel.save_pretrained`] or upload it to the Hub with the [`~transformers.PreTrainedModel.push_to_hub`] method.\n\n```py\n# save locally\nlora_model.save_pretrained(\"your-name/opt-350m-lora\")\n\n# push to Hub\nlora_model.push_to_hub(\"your-name/opt-350m-lora\")\n```\n\nTo load a [`PeftModel`] for inference, you'll need to provide the [`PeftConfig`] used to create it and the base model it was trained from."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 52ef35f5-5b16-4a70-8784-f0035edabc3a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/tutorial/peft_model_config", "title": "PEFT configurations and models", "tokens": 1940, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"52ef35f5-5b16-4a70-8784-f0035edabc3a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"5ac96aa4-9551-4087-b933-4d09cbf3fd9b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"36f7eede49bcbb1fbca50686517a245cfc6d4d7d3a86c2d0d5cb453d0dbc1f4a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8987281f-4cf5-4e7c-bf8b-50c6243d967a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/tutorial/peft_model_config\", \"title\": \"PEFT configurations and models\", \"tokens\": 1940, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"7111e30ebfd35b4c6c3358ba550f431effe1cca95507cd67214639c93cfe1b73\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3970, \"end_char_idx\": 6903, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "ref_doc_id": "5ac96aa4-9551-4087-b933-4d09cbf3fd9b", "chroma:document": "Choose from any of the state-of-the-art models from the [Transformers](https://hf.co/docs/transformers) library, a custom model, and even new and unsupported transformer architectures.\n\nFor this tutorial, load a base [facebook/opt-350m](https://huggingface.co/facebook/opt-350m) model to finetune.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\")\n```\n\nUse the [`get_peft_model`] function to create a [`PeftModel`] from the base facebook/opt-350m model and the `lora_config` you created earlier.\n\n```py\nfrom peft import get_peft_model\n\nlora_model = get_peft_model(model, lora_config)\nlora_model.print_trainable_parameters()\n\"trainable params: 1,572,864 || all params: 332,769,280 || trainable%: 0.472659014678278\"\n```\n\nNow you can train the [`PeftModel`] with your preferred training framework! After training, you can save your model locally with [`~PeftModel.save_pretrained`] or upload it to the Hub with the [`~transformers.PreTrainedModel.push_to_hub`] method.\n\n```py\n# save locally\nlora_model.save_pretrained(\"your-name/opt-350m-lora\")\n\n# push to Hub\nlora_model.push_to_hub(\"your-name/opt-350m-lora\")\n```\n\nTo load a [`PeftModel`] for inference, you'll need to provide the [`PeftConfig`] used to create it and the base model it was trained from.\n\n```py\nfrom peft import PeftModel, PeftConfig\n\nconfig = PeftConfig.from_pretrained(\"ybelkada/opt-350m-lora\")\nmodel = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)\nlora_model = PeftModel.from_pretrained(model, \"ybelkada/opt-350m-lora\")\n```\n\n<Tip>\n\nBy default, the [`PeftModel`] is set for inference, but if you'd like to train the adapter some more you can set `is_trainable=True`.\n\n```py\nlora_model = PeftModel.from_pretrained(model, \"ybelkada/opt-350m-lora\", is_trainable=True)\n```\n\n</Tip>\n\nThe [`PeftModel.from_pretrained`] method is the most flexible way to load a [`PeftModel`] because it doesn't matter what model framework was used (Transformers, timm, a generic PyTorch model). Other classes, like [`AutoPeftModel`], are just a convenient wrapper around the base [`PeftModel`], and makes it easier to load PEFT models directly from the Hub or locally where the PEFT weights are stored.\n\n```py\nfrom peft import AutoPeftModelForCausalLM\n\nlora_model = AutoPeftModelForCausalLM.from_pretrained(\"ybelkada/opt-350m-lora\")\n```\n\nTake a look at the [AutoPeftModel](package_reference/auto_class) API reference to learn more about the [`AutoPeftModel`] classes.\n\n## Next steps\n\nWith the appropriate [`PeftConfig`], you can apply it to any pretrained model to create a [`PeftModel`] and train large powerful models faster on freely available GPUs! To learn more about PEFT configurations and models, the following guide may be helpful:\n\n* Learn how to configure a PEFT method for models that aren't from Transformers in the [Working with custom models](../developer_guides/custom_models) guide."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: a49b054c-493a-4e0e-9ce5-b8fb25708755
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/peft_model", "title": "Models", "tokens": 452, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"a49b054c-493a-4e0e-9ce5-b8fb25708755\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/peft_model\", \"title\": \"Models\", \"tokens\": 452, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6e670844-7df2-49a1-971c-ab08f6ee6bc7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/peft_model\", \"title\": \"Models\", \"tokens\": 452, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"734e54070125d80cc73779d82b70dcc5fc23e84b9a6654f7dd9335368bad14d0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1670, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6e670844-7df2-49a1-971c-ab08f6ee6bc7", "doc_id": "6e670844-7df2-49a1-971c-ab08f6ee6bc7", "ref_doc_id": "6e670844-7df2-49a1-971c-ab08f6ee6bc7", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Models\n\n[`PeftModel`] is the base model class for specifying the base Transformer model and configuration to apply a PEFT method to. The base `PeftModel` contains methods for loading and saving models from the Hub.\n\n## PeftModel\n\n[[autodoc]] PeftModel\n    - all\n\n## PeftModelForSequenceClassification\n\nA `PeftModel` for sequence classification tasks.\n\n[[autodoc]] PeftModelForSequenceClassification\n    - all\n\n## PeftModelForTokenClassification\n\nA `PeftModel` for token classification tasks.\n\n[[autodoc]] PeftModelForTokenClassification\n    - all\n\n## PeftModelForCausalLM\n\nA `PeftModel` for causal language modeling.\n\n[[autodoc]] PeftModelForCausalLM\n    - all\n\n## PeftModelForSeq2SeqLM\n\nA `PeftModel` for sequence-to-sequence language modeling.\n\n[[autodoc]] PeftModelForSeq2SeqLM\n    - all\n\n## PeftModelForQuestionAnswering\n\nA `PeftModel` for question answering.\n\n[[autodoc]] PeftModelForQuestionAnswering\n    - all\n\n## PeftModelForFeatureExtraction\n\nA `PeftModel` for getting extracting features/embeddings from transformer models.\n\n[[autodoc]] PeftModelForFeatureExtraction\n    - all\n\n## PeftMixedModel\n\nA `PeftModel` for mixing different adapter types (e.g. LoRA and LoHa).\n\n[[autodoc]] PeftMixedModel\n    - all\n\n## Utilities\n\n[[autodoc]] utils.cast_mixed_precision_params\n\n[[autodoc]] get_peft_model\n\n[[autodoc]] inject_adapter_in_model\n\n[[autodoc]] utils.get_peft_model_state_dict\n\n[[autodoc]] utils.prepare_model_for_kbit_training\n\n[[autodoc]] get_layer_status\n\n[[autodoc]] get_model_status"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: a8dc2ba3-be76-4fba-8d37-39b9f874b56e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/boft", "title": "BOFT", "tokens": 538, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"a8dc2ba3-be76-4fba-8d37-39b9f874b56e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/boft\", \"title\": \"BOFT\", \"tokens\": 538, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c935c2c8-095e-4e9d-9c2b-98759c7b5c14\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/boft\", \"title\": \"BOFT\", \"tokens\": 538, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"cbf4acf59d6db8eaf5d779b95e124476463b2e388b7d49bb45dcf30e9c962275\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1884, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c935c2c8-095e-4e9d-9c2b-98759c7b5c14", "doc_id": "c935c2c8-095e-4e9d-9c2b-98759c7b5c14", "ref_doc_id": "c935c2c8-095e-4e9d-9c2b-98759c7b5c14", "chroma:document": "# BOFT\n\n[Orthogonal Butterfly (BOFT)](https://hf.co/papers/2311.06243) is a generic method designed for finetuning foundation models. It improves the paramter efficiency of the finetuning paradigm -- Orthogonal Finetuning (OFT), by taking inspiration from Cooley-Tukey fast Fourier transform, showing favorable results across finetuning different foundation models, including large vision transformers, large language models and text-to-image diffusion models.\n\nThe abstract from the paper is:\n\n*Large foundation models are becoming ubiquitous, but training them from scratch is prohibitively expensive. Thus, efficiently adapting these powerful models to downstream tasks is increasingly important. In this paper, we study a principled finetuning paradigm -- Orthogonal Finetuning (OFT) -- for downstream task adaptation. Despite demonstrating good generalizability, OFT still uses a fairly large number of trainable parameters due to the high dimensionality of orthogonal matrices. To address this, we start by examining OFT from an information transmission perspective, and then identify a few key desiderata that enable better parameter-efficiency. Inspired by how the Cooley-Tukey fast Fourier transform algorithm enables efficient information transmission, we propose an efficient orthogonal parameterization using butterfly structures. We apply this parameterization to OFT, creating a novel parameter-efficient finetuning method, called Orthogonal Butterfly (BOFT). By subsuming OFT as a special case, BOFT introduces a generalized orthogonal finetuning framework. Finally, we conduct an extensive empirical study of adapting large vision transformers, large language models, and text-to-image diffusion models to various downstream tasks in vision and language*.\n\n## BOFTConfig\n\n[[autodoc]] tuners.boft.config.BOFTConfig\n\n## BOFTModel\n\n[[autodoc]] tuners.boft.model.BOFTModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: abef8b70-4699-4480-b6ca-e066c5cf21ec
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/llama_adapter", "title": "Llama-Adapter", "tokens": 544, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"abef8b70-4699-4480-b6ca-e066c5cf21ec\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/llama_adapter\", \"title\": \"Llama-Adapter\", \"tokens\": 544, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4c902989-05b8-42e4-9692-6032b2f6efcb\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/llama_adapter\", \"title\": \"Llama-Adapter\", \"tokens\": 544, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a57dc51f189af4a02b19a5a32f632c9e84fed3becfb7fa8800963255eed8ea0e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1734, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4c902989-05b8-42e4-9692-6032b2f6efcb", "doc_id": "4c902989-05b8-42e4-9692-6032b2f6efcb", "ref_doc_id": "4c902989-05b8-42e4-9692-6032b2f6efcb", "chroma:document": "# Llama-Adapter\n\n[Llama-Adapter](https://hf.co/papers/2303.16199) is a PEFT method specifically designed for turning Llama into an instruction-following model. The Llama model is frozen and only a set of adaptation prompts prefixed to the input instruction tokens are learned. Since randomly initialized modules inserted into the model can cause the model to lose some of its existing knowledge, Llama-Adapter uses zero-initialized attention with zero gating to progressively add the instructional prompts to the model.\n\nThe abstract from the paper is:\n\n*We present LLaMA-Adapter, a lightweight adaption method to efficiently fine-tune LLaMA into an instruction-following model. Using 52K self-instruct demonstrations, LLaMA-Adapter only introduces 1.2M learnable parameters upon the frozen LLaMA 7B model, and costs less than one hour for fine-tuning on 8 A100 GPUs. Specifically, we adopt a set of learnable adaption prompts, and prepend them to the input text tokens at higher transformer layers. Then, a zero-init attention mechanism with zero gating is proposed, which adaptively injects the new instructional cues into LLaMA, while effectively preserves its pre-trained knowledge. With efficient training, LLaMA-Adapter generates high-quality responses, comparable to Alpaca with fully fine-tuned 7B parameters. Furthermore, our approach can be simply extended to multi-modal input, e.g., images, for image-conditioned LLaMA, which achieves superior reasoning capacity on ScienceQA. We release our code at https://github.com/ZrrSkywalker/LLaMA-Adapter*.\n\n## AdaptionPromptConfig\n\n[[autodoc]] tuners.adaption_prompt.config.AdaptionPromptConfig\n\n## AdaptionPromptModel\n\n[[autodoc]] tuners.adaption_prompt.model.AdaptionPromptModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 2b1a76e4-5d4e-4824-944b-d2ba18e50ca8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/layernorm_tuning", "title": "LayerNorm Tuning", "tokens": 609, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"2b1a76e4-5d4e-4824-944b-d2ba18e50ca8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/layernorm_tuning\", \"title\": \"LayerNorm Tuning\", \"tokens\": 609, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"803bf356-3de4-4a1b-87d3-b76c65fd6629\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/layernorm_tuning\", \"title\": \"LayerNorm Tuning\", \"tokens\": 609, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"6751ef8af926d71033019b5632956d6a8aa167a771ea0d59e9105ba193488efa\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2080, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "803bf356-3de4-4a1b-87d3-b76c65fd6629", "doc_id": "803bf356-3de4-4a1b-87d3-b76c65fd6629", "ref_doc_id": "803bf356-3de4-4a1b-87d3-b76c65fd6629", "chroma:document": "# LayerNorm Tuning\n\nLayerNorm Tuning ([LN Tuning](https://huggingface.co/papers/2312.11420)) is a PEFT method that only fine-tunes the parameters of the LayerNorm layers in a model.\nThe paper has tested the performance of this method on large language models and has shown that it can achieve strong performance with a significant reduction in the number of trainable parameters and GPU memory usage.\nHowever, the method is not limited to language models and can be applied to any model that uses LayerNorm layers.\nIn this implementation, the default is that all layernorm layers inside a model is finetuned, but it could be used to target other layer types such as `MLP` or `Attention` layers, this can be done by specifying the `target_modules` in the `LNTuningConfig`.\n\nThe abstract from the paper is:\n\n*This paper introduces an efficient strategy to transform Large Language Models (LLMs) into Multi-Modal Large Language Models (MLLMs). By conceptualizing this transformation as a domain adaptation process, i.e., transitioning from text understanding to embracing multiple modalities, we intriguingly note that, within each attention block, tuning LayerNorm suffices to yield strong performance. Moreover, when benchmarked against other tuning approaches like full parameter finetuning or LoRA, its benefits on efficiency are substantial. For example, when compared to LoRA on a 13B model scale, performance can be enhanced by an average of over 20% across five multi-modal tasks, and meanwhile, results in a significant reduction of trainable parameters by 41.9% and a decrease in GPU memory usage by 17.6%. On top of this LayerNorm strategy, we showcase that selectively tuning only with conversational data can improve efficiency further. Beyond these empirical outcomes, we provide a comprehensive analysis to explore the role of LayerNorm in adapting LLMs to the multi-modal domain and improving the expressive power of the model.*\n\n## LNTuningConfig\n\n[[autodoc]] tuners.ln_tuning.config.LNTuningConfig\n\n## LNTuningModel\n\n[[autodoc]] tuners.ln_tuning.model.LNTuningModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 74de2dea-b648-412a-8af7-66c14e35ef05
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/ia3", "title": "IA3", "tokens": 558, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"74de2dea-b648-412a-8af7-66c14e35ef05\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/ia3\", \"title\": \"IA3\", \"tokens\": 558, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"77667445-b877-40e0-b91a-f0060f68a08f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/ia3\", \"title\": \"IA3\", \"tokens\": 558, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"636416b9c7d25072c712deb67eedc25f3573df7c1f03e7a608e2dad00b2ddb4f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1848, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "77667445-b877-40e0-b91a-f0060f68a08f", "doc_id": "77667445-b877-40e0-b91a-f0060f68a08f", "ref_doc_id": "77667445-b877-40e0-b91a-f0060f68a08f", "chroma:document": "# IA3\n\nInfused Adapter by Inhibiting and Amplifying Inner Activations, or [IA3](https://hf.co/papers/2205.05638), is a method that adds three learned vectors to rescale the keys and values of the self-attention and encoder-decoder attention layers, and the intermediate activation of the position-wise feed-forward network.\n\nThe abstract from the paper is:\n\n*Few-shot in-context learning (ICL) enables pre-trained language models to perform a previously-unseen task without any gradient-based training by feeding a small number of training examples as part of the input. ICL incurs substantial computational, memory, and storage costs because it involves processing all of the training examples every time a prediction is made. Parameter-efficient fine-tuning (PEFT) (e.g. adapter modules, prompt tuning, sparse update methods, etc.) offers an alternative paradigm where a small set of parameters are trained to enable a model to perform the new task. In this paper, we rigorously compare few-shot ICL and PEFT and demonstrate that the latter offers better accuracy as well as dramatically lower computational costs. Along the way, we introduce a new PEFT method called (IA)^3 that scales activations by learned vectors, attaining stronger performance while only introducing a relatively tiny amount of new parameters. We also propose a simple recipe based on the T0 model called T-Few that can be applied to new tasks without task-specific tuning or modifications. We validate the effectiveness of T-Few on completely unseen tasks by applying it to the RAFT benchmark, attaining super-human performance for the first time and outperforming the state-of-the-art by 6% absolute. All of the code used in our experiments is publicly available*.\n\n## IA3Config\n\n[[autodoc]] tuners.ia3.config.IA3Config\n\n## IA3Model\n\n[[autodoc]] tuners.ia3.model.IA3Model"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 86c15c0e-3765-4eb9-92ac-61d72b02a70d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/adalora", "title": "AdaLoRA", "tokens": 562, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"86c15c0e-3765-4eb9-92ac-61d72b02a70d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/adalora\", \"title\": \"AdaLoRA\", \"tokens\": 562, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"993b2fe2-1f41-48eb-b877-f2f925af9f3a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/adalora\", \"title\": \"AdaLoRA\", \"tokens\": 562, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8ba439ee61d8e3052dc4f28ae0693cb039774edf1c76e98b0a05f6b6163da63a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2017, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "993b2fe2-1f41-48eb-b877-f2f925af9f3a", "doc_id": "993b2fe2-1f41-48eb-b877-f2f925af9f3a", "ref_doc_id": "993b2fe2-1f41-48eb-b877-f2f925af9f3a", "chroma:document": "# AdaLoRA\n\n[AdaLoRA](https://hf.co/papers/2303.10512) is a method for optimizing the number of trainable parameters to assign to weight matrices and layers, unlike LoRA, which distributes parameters evenly across all modules. More parameters are budgeted for important weight matrices and layers while less important ones receive fewer parameters.\n\nThe abstract from the paper is:\n\n*Fine-tuning large pre-trained language models on downstream tasks has become an important paradigm in NLP. However, common practice fine-tunes all of the parameters in a pre-trained model, which becomes prohibitive when a large number of downstream tasks are present. Therefore, many fine-tuning methods are proposed to learn incremental updates of pre-trained weights in a parameter efficient way, e.g., low-rank increments. These methods often evenly distribute the budget of incremental updates across all pre-trained weight matrices, and overlook the varying importance of different weight parameters. As a consequence, the fine-tuning performance is suboptimal. To bridge this gap, we propose AdaLoRA, which adaptively allocates the parameter budget among weight matrices according to their importance score. In particular, AdaLoRA parameterizes the incremental updates in the form of singular value decomposition. Such a novel approach allows us to effectively prune the singular values of unimportant updates, which is essentially to reduce their parameter budget but circumvent intensive exact SVD computations. We conduct extensive experiments with several pre-trained models on natural language processing, question answering, and natural language generation to validate the effectiveness of AdaLoRA. Results demonstrate that AdaLoRA manifests notable improvement over baselines, especially in the low budget settings. Our code is publicly available at https://github.com/QingruZhang/AdaLoRA*.\n\n## AdaLoraConfig\n\n[[autodoc]] tuners.adalora.config.AdaLoraConfig\n\n## AdaLoraModel\n\n[[autodoc]] tuners.adalora.model.AdaLoraModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: c28e84e1-ed03-40a1-98d1-1594552fafe8
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/xlora", "title": "X-LoRA", "tokens": 1284, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"c28e84e1-ed03-40a1-98d1-1594552fafe8\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/xlora\", \"title\": \"X-LoRA\", \"tokens\": 1284, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"097a1790-f75c-45d1-b07e-b5f183e1cf2a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/xlora\", \"title\": \"X-LoRA\", \"tokens\": 1284, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"6b0845ba736a95fda87f9b1cccdf24a18aa85611def75c4e27e8541b7fbd5810\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"883fae9f-62e8-4a70-aa68-a8992b904889\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"eaea626b377c5eb51f4957f9482b451ed4c4ff528feec0acf5d7c0e99d6da0c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3663, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "doc_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "ref_doc_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "chroma:document": "# X-LoRA\n\nMixture of LoRA Experts ([X-LoRA](https://arxiv.org/abs/2402.07148)) is a PEFT method enabling sparse or dense mixture of LoRA experts based on a high granularity (token, layer, sequence) scalings matrix. This leverages frozen LoRA adapters and a frozen base model to drastically reduces the number of parameters that need to be fine-tuned.\n\nA unique aspect of X-LoRA is its versatility: it can be applied to any `transformers` base model with LoRA adapters. This means that, despite the mixture of experts strategy, no changes to the model code must be made.\n\nThe below graphic demonstrates how the scalings change for different prompts for each token. This highlights the activation of different adapters as the generation progresses and the sequence creates new context.\n\n![Token-by-token scalings](https://github.com/EricLBuehler/xlora/raw/master/res/token_by_token_scalings.gif)\n\nThe abstract from the paper is:\n\n*We report a mixture of expert strategy to create fine-tuned large language models using a deep layer-wise token-level approach based on low-rank adaptation (LoRA). Starting with a set of pre-trained LoRA adapters, our gating strategy uses the hidden states to dynamically mix adapted layers, allowing the resulting X-LoRA model to draw upon different capabilities and create never-before-used deep layer-wise combinations to solve tasks. The design is inspired by the biological principles of universality and diversity, where neural network building blocks are reused in different hierarchical manifestations. Hence, the X-LoRA model can be easily implemented for any existing large language model (LLM) without a need for modifications of the underlying structure. We develop a tailored X-LoRA model that offers scientific capabilities including forward/inverse analysis tasks and enhanced reasoning capability, focused on biomaterial analysis, protein mechanics and design. The impact of this work include access to readily expandable and adaptable models with strong domain knowledge and the capability to integrate across areas of knowledge. Featuring experts in biology, mathematics, reasoning, bio-inspired materials, mechanics and materials, chemistry, protein biophysics, mechanics and quantum-mechanics based molecular properties, we conduct a series of physics-focused case studies. We examine knowledge recall, protein mechanics forward/inverse tasks, protein design, adversarial agentic modeling including ontological knowledge graph construction, as well as molecular design. The model is capable not only of making quantitative predictions of nanomechanical properties of proteins or quantum mechanical molecular properties, but also reasons over the results and correctly predicts likely mechanisms that explain distinct molecular behaviors.*.\n\nPlease cite X-LoRA as:\n```bibtex\n@article{10.1063/5.0203126,\n    author = {Buehler, Eric L. and Buehler, Markus J.},\n    title = \"{X-LoRA: Mixture of low-rank adapter experts, a flexible framework for large language models with applications in protein mechanics and molecular design}\",\n    journal = {APL Machine Learning},\n    volume = {2},\n    number = {2},\n    pages = {026119},\n    year = {2024},\n    month = {05},\n    abstract = \"{We report a mixture of expert strategy to create fine-tuned large language models using a deep layer-wise token-level approach based on low-rank adaptation (LoRA). Starting with a set of pre-trained LoRA adapters, our gating strategy uses the hidden states to dynamically mix adapted layers, allowing the resulting X-LoRA model to draw upon different capabilities and create never-before-used deep layer-wise combinations to solve tasks."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 883fae9f-62e8-4a70-aa68-a8992b904889
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/xlora", "title": "X-LoRA", "tokens": 1284, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"883fae9f-62e8-4a70-aa68-a8992b904889\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/xlora\", \"title\": \"X-LoRA\", \"tokens\": 1284, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"097a1790-f75c-45d1-b07e-b5f183e1cf2a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/xlora\", \"title\": \"X-LoRA\", \"tokens\": 1284, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"6b0845ba736a95fda87f9b1cccdf24a18aa85611def75c4e27e8541b7fbd5810\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c28e84e1-ed03-40a1-98d1-1594552fafe8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/xlora\", \"title\": \"X-LoRA\", \"tokens\": 1284, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a7c5b0d23337014c5c844e974d696065d89eff79485fed445bb719da5f8b2997\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1906, \"end_char_idx\": 5420, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "doc_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "ref_doc_id": "097a1790-f75c-45d1-b07e-b5f183e1cf2a", "chroma:document": "The impact of this work include access to readily expandable and adaptable models with strong domain knowledge and the capability to integrate across areas of knowledge. Featuring experts in biology, mathematics, reasoning, bio-inspired materials, mechanics and materials, chemistry, protein biophysics, mechanics and quantum-mechanics based molecular properties, we conduct a series of physics-focused case studies. We examine knowledge recall, protein mechanics forward/inverse tasks, protein design, adversarial agentic modeling including ontological knowledge graph construction, as well as molecular design. The model is capable not only of making quantitative predictions of nanomechanical properties of proteins or quantum mechanical molecular properties, but also reasons over the results and correctly predicts likely mechanisms that explain distinct molecular behaviors.*.\n\nPlease cite X-LoRA as:\n```bibtex\n@article{10.1063/5.0203126,\n    author = {Buehler, Eric L. and Buehler, Markus J.},\n    title = \"{X-LoRA: Mixture of low-rank adapter experts, a flexible framework for large language models with applications in protein mechanics and molecular design}\",\n    journal = {APL Machine Learning},\n    volume = {2},\n    number = {2},\n    pages = {026119},\n    year = {2024},\n    month = {05},\n    abstract = \"{We report a mixture of expert strategy to create fine-tuned large language models using a deep layer-wise token-level approach based on low-rank adaptation (LoRA). Starting with a set of pre-trained LoRA adapters, our gating strategy uses the hidden states to dynamically mix adapted layers, allowing the resulting X-LoRA model to draw upon different capabilities and create never-before-used deep layer-wise combinations to solve tasks. The design is inspired by the biological principles of universality and diversity, where neural network building blocks are reused in different hierarchical manifestations. Hence, the X-LoRA model can be easily implemented for any existing large language model without a need for modifications of the underlying structure. We develop a tailored X-LoRA model that offers scientific capabilities, including forward/inverse analysis tasks and enhanced reasoning capability, focused on biomaterial analysis, protein mechanics, and design. The impact of this work includes access to readily expandable and adaptable models with strong domain knowledge and the capability to integrate across areas of knowledge. Featuring experts in biology, mathematics, reasoning, bio-inspired materials, mechanics and materials, chemistry, protein biophysics, mechanics, and quantum-mechanics based molecular properties, we conduct a series of physics-focused case studies. We examine knowledge recall, protein mechanics forward/inverse tasks, protein design, adversarial agentic modeling including ontological knowledge graph construction, and molecular design. The model is capable not only of making quantitative predictions of nanomechanical properties of proteins or quantum mechanical molecular properties but also reasoning over the results and correctly predicting likely mechanisms that explain distinct molecular behaviors.}\",\n    issn = {2770-9019},\n    doi = {10.1063/5.0203126},\n    url = {https://doi.org/10.1063/5.0203126},\n    eprint = {https://pubs.aip.org/aip/aml/article-pdf/doi/10.1063/5.0203126/19964043/026119\\_1\\_5.0203126.pdf},\n}\n```\n\n## XLoraConfig\n\n[[autodoc]] tuners.xlora.config.XLoraConfig\n\n## XLoraModel\n\n[[autodoc]] tuners.xlora.model.XLoraModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 3b9ab0f2-e8e6-4f16-878a-7111d0f73349
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/helpers", "title": "Helper methods", "tokens": 107, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"3b9ab0f2-e8e6-4f16-878a-7111d0f73349\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/helpers\", \"title\": \"Helper methods\", \"tokens\": 107, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0dda1314-910d-47b8-bfa9-35f9e27f441e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/helpers\", \"title\": \"Helper methods\", \"tokens\": 107, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"6230b09b5ce5c6ac37770c94973a334cf128c033a017de7f596baeae448a5c68\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 436, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0dda1314-910d-47b8-bfa9-35f9e27f441e", "doc_id": "0dda1314-910d-47b8-bfa9-35f9e27f441e", "ref_doc_id": "0dda1314-910d-47b8-bfa9-35f9e27f441e", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Helper methods\n\nA collection of helper functions for PEFT.\n\n## Checking if a model is a PEFT model\n\n[[autodoc]] helpers.check_if_peft_model\n    - all\n\n## Temporarily Rescaling Adapter Scale in LoraLayer Modules\n\n[[autodoc]] helpers.rescale_adapter_scale\n    - all"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: a80f2e21-814e-464e-9e7c-a4fd370d0628
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/prompt_tuning", "title": "Prompt tuning", "tokens": 504, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"a80f2e21-814e-464e-9e7c-a4fd370d0628\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/prompt_tuning\", \"title\": \"Prompt tuning\", \"tokens\": 504, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0165aaf1-d1b4-4e91-b2f0-65ad87c2a410\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/prompt_tuning\", \"title\": \"Prompt tuning\", \"tokens\": 504, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"c0f3438301e0873a09ad0d7df1866e8850298b4c9bad50cb7315cf14a0df6feb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1638, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0165aaf1-d1b4-4e91-b2f0-65ad87c2a410", "doc_id": "0165aaf1-d1b4-4e91-b2f0-65ad87c2a410", "ref_doc_id": "0165aaf1-d1b4-4e91-b2f0-65ad87c2a410", "chroma:document": "# Prompt tuning\n\n[Prompt tuning](https://hf.co/papers/2104.08691) adds task-specific prompts to the input, and these prompt parameters are updated independently of the pretrained model parameters which are frozen.\n\nThe abstract from the paper is:\n\n*In this work, we explore \"prompt tuning\", a simple yet effective mechanism for learning \"soft prompts\" to condition frozen language models to perform specific downstream tasks. Unlike the discrete text prompts used by GPT-3, soft prompts are learned through backpropagation and can be tuned to incorporate signal from any number of labeled examples. Our end-to-end learned approach outperforms GPT-3's \"few-shot\" learning by a large margin. More remarkably, through ablations on model size using T5, we show that prompt tuning becomes more competitive with scale: as models exceed billions of parameters, our method \"closes the gap\" and matches the strong performance of model tuning (where all model weights are tuned). This finding is especially relevant in that large models are costly to share and serve, and the ability to reuse one frozen model for multiple downstream tasks can ease this burden. Our method can be seen as a simplification of the recently proposed \"prefix tuning\" of Li and Liang (2021), and we provide a comparison to this and other similar approaches. Finally, we show that conditioning a frozen model with soft prompts confers benefits in robustness to domain transfer, as compared to full model tuning*.\n\n## PromptTuningConfig\n\n[[autodoc]] tuners.prompt_tuning.config.PromptTuningConfig\n\n## PromptEmbedding\n\n[[autodoc]] tuners.prompt_tuning.model.PromptEmbedding"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 2a5f7dbe-db15-42fd-bbe2-058203c04e1d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/tuners", "title": "Tuners", "tokens": 288, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"2a5f7dbe-db15-42fd-bbe2-058203c04e1d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/tuners\", \"title\": \"Tuners\", \"tokens\": 288, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a9af5033-7494-4938-b2b0-285bd965b29e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/tuners\", \"title\": \"Tuners\", \"tokens\": 288, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"45573e9e41c717a7b2cd5b1a3da833079bbfe376b6c0e7ff55ed72cf029ba4ca\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 544, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a9af5033-7494-4938-b2b0-285bd965b29e", "doc_id": "a9af5033-7494-4938-b2b0-285bd965b29e", "ref_doc_id": "a9af5033-7494-4938-b2b0-285bd965b29e", "chroma:document": "# Tuners\n\nA tuner (or adapter) is a module that can be plugged into a `torch.nn.Module`. [`BaseTuner`] base class for other tuners and provides shared methods and attributes for preparing an adapter configuration and replacing a target module with the adapter module. [`BaseTunerLayer`] is a base class for adapter layers. It offers methods and attributes for managing adapters such as activating and disabling adapters.\n\n## BaseTuner\n\n[[autodoc]] tuners.tuners_utils.BaseTuner\n\n## BaseTunerLayer\n\n[[autodoc]] tuners.tuners_utils.BaseTunerLayer"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d58ee32d-281d-4dec-a622-0282cbcd2878
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/multitask_prompt_tuning", "title": "Multitask prompt tuning", "tokens": 475, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d58ee32d-281d-4dec-a622-0282cbcd2878\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/multitask_prompt_tuning\", \"title\": \"Multitask prompt tuning\", \"tokens\": 475, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"b44b7b2d-5568-4cbb-adcd-b108682c48a1\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/multitask_prompt_tuning\", \"title\": \"Multitask prompt tuning\", \"tokens\": 475, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"4eec57590220074b8942f8b61bc742a608ec681de263aadac8e681da749c02bc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1512, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "b44b7b2d-5568-4cbb-adcd-b108682c48a1", "doc_id": "b44b7b2d-5568-4cbb-adcd-b108682c48a1", "ref_doc_id": "b44b7b2d-5568-4cbb-adcd-b108682c48a1", "chroma:document": "# Multitask prompt tuning\n\n[Multitask prompt tuning](https://huggingface.co/papers/2303.02861)  decomposes the soft prompts of each task into a single learned transferable prompt instead of a separate prompt for each task. The single learned prompt can be adapted for each task by multiplicative low rank updates.\n\nThe abstract from the paper is:\n\n*Prompt tuning, in which a base pretrained model is adapted to each task via conditioning on learned prompt vectors, has emerged as a promising approach for efficiently adapting large language models to multiple downstream tasks. However, existing methods typically learn soft prompt vectors from scratch, and it has not been clear how to exploit the rich cross-task knowledge with prompt vectors in a multitask learning setting. We propose multitask prompt tuning (MPT), which first learns a single transferable prompt by distilling knowledge from multiple task-specific source prompts. We then learn multiplicative low rank updates to this shared prompt to efficiently adapt it to each downstream target task. Extensive experiments on 23 NLP datasets demonstrate that our proposed approach outperforms the state-of-the-art methods, including the full finetuning baseline in some cases, despite only tuning 0.035% as many task-specific parameters*.\n\n## MultitaskPromptTuningConfig\n\n[[autodoc]] tuners.multitask_prompt_tuning.config.MultitaskPromptTuningConfig\n\n## MultitaskPromptEmbedding\n\n[[autodoc]] tuners.multitask_prompt_tuning.model.MultitaskPromptEmbedding"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 9ac42d65-05e3-4b43-8aec-858e5bb00447
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/lora", "title": "LoRA", "tokens": 467, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"9ac42d65-05e3-4b43-8aec-858e5bb00447\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/lora\", \"title\": \"LoRA\", \"tokens\": 467, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1d068454-a4d0-4e93-8440-5789a36dc1ef\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/lora\", \"title\": \"LoRA\", \"tokens\": 467, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"0cfcc6e4a37b702c4db63008864e82d6a9cb07b713a7cb6e3a8bd2eed23f2c86\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1364, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1d068454-a4d0-4e93-8440-5789a36dc1ef", "doc_id": "1d068454-a4d0-4e93-8440-5789a36dc1ef", "ref_doc_id": "1d068454-a4d0-4e93-8440-5789a36dc1ef", "chroma:document": "# LoRA\n\nLow-Rank Adaptation ([LoRA](https://huggingface.co/papers/2309.15223)) is a PEFT method that decomposes a large matrix into two smaller low-rank matrices in the attention layers. This drastically reduces the number of parameters that need to be fine-tuned.\n\nThe abstract from the paper is:\n\n*We propose a neural language modeling system based on low-rank adaptation (LoRA) for speech recognition output rescoring. Although pretrained language models (LMs) like BERT have shown superior performance in second-pass rescoring, the high computational cost of scaling up the pretraining stage and adapting the pretrained models to specific domains limit their practical use in rescoring. Here we present a method based on low-rank decomposition to train a rescoring BERT model and adapt it to new domains using only a fraction (0.08%) of the pretrained parameters. These inserted matrices are optimized through a discriminative training objective along with a correlation-based regularization loss. The proposed low-rank adaptation Rescore-BERT (LoRB) architecture is evaluated on LibriSpeech and internal datasets with decreased training times by factors between 5.4 and 3.6.*.\n\n## LoraConfig\n\n[[autodoc]] tuners.lora.config.LoraConfig\n\n## LoraModel\n\n[[autodoc]] tuners.lora.model.LoraModel\n\n## Utility\n\n[[autodoc]] utils.loftq_utils.replace_lora_weights_loftq"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d30441c2-d50f-4f69-aee3-900bf86df281
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/vera", "title": "VeRA: Vector-based Random Matrix Adaptation", "tokens": 756, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d30441c2-d50f-4f69-aee3-900bf86df281\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/vera\", \"title\": \"VeRA: Vector-based Random Matrix Adaptation\", \"tokens\": 756, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d3416c50-2ddb-46e8-85d2-180d231064a5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/vera\", \"title\": \"VeRA: Vector-based Random Matrix Adaptation\", \"tokens\": 756, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a966022ee2c4040c9e940a76708928ed3402c488a512b5ded5c1d4d3b65d65b4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2700, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d3416c50-2ddb-46e8-85d2-180d231064a5", "doc_id": "d3416c50-2ddb-46e8-85d2-180d231064a5", "ref_doc_id": "d3416c50-2ddb-46e8-85d2-180d231064a5", "chroma:document": "# VeRA: Vector-based Random Matrix Adaptation\n\n[VeRA](https://huggingface.co/papers/2310.11454) is a parameter-efficient fine-tuning technique that is similar to LoRA but requires even fewer extra parameters while promising similar or even better performance. As such, it is particularly useful when the parameter budget is very limited, e.g. when scaling to very large models. The reduction of the count of trainable parameters is achieved by sharing the same low-rank matrices across all layers, and only training two additional vectors per layer.\n\nWhen saving the adapter parameters, it's possible to eschew storing the low rank matrices by setting `save_projection=False` on the `VeraConfig`. In that case, these matrices will be restored based on the fixed random seed from the `projection_prng_key` argument. This cuts down on the size of the checkpoint, but we cannot guarantee reproducibility on all devices and for all future versions of PyTorch. If you want to ensure reproducibility, set `save_projection=True` (which is the default).\n\nTo handle different shapes of adapted layers, VeRA initializes shared A and B matrices with the largest required size for each dimension. During the forward pass, submatrices A and B for a given layer are sliced out from these shared matrices and used as described in the paper. For example, adapting two linear layers of shapes (100, 20) and (80, 50) will create A and B matrices of shapes (rank, 50) and (100, rank) respectively. Then, to adapt a layer of shape (100, 20), submatrices A and B of shapes (rank, 20) and (100, rank) will be extracted.\n\nVeRA currently has the following constraints:\n\n- Only `nn.Linear` layers are supported.\n- Quantized layers are not supported.\n\nIf these constraints don't work for your use case, use LoRA instead.\n\nThe abstract from the paper is:\n\n> Low-rank adapation (LoRA) is a popular method that reduces the number of trainable parameters when finetuning large language models, but still faces acute storage challenges when scaling to even larger models or deploying numerous per-user or per-task adapted models. In this work, we present Vector-based Random Matrix Adaptation (VeRA), which significantly reduces the number of trainable parameters compared to LoRA, yet maintains the same performance. It achieves this by using a single pair of low-rank matrices shared across all layers and learning small scaling vectors instead. We demonstrate its effectiveness on the GLUE and E2E benchmarks, image classification tasks, and show its application in instruction-tuning of 7B and 13B language models.\n\n## VeRAConfig\n\n[[autodoc]] tuners.vera.config.VeraConfig\n\n## VeRAModel\n\n[[autodoc]] tuners.vera.model.VeraModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 20c4b079-3809-4e39-88f2-97a373828d2e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/config", "title": "Configuration", "tokens": 193, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"20c4b079-3809-4e39-88f2-97a373828d2e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/config\", \"title\": \"Configuration\", \"tokens\": 193, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d1ed01e5-ec86-4e42-8791-637e437bbe8e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/config\", \"title\": \"Configuration\", \"tokens\": 193, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3a6c33d6e8f128a6679644783bea19d306625bb345a62906da6b0395b598d282\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 843, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d1ed01e5-ec86-4e42-8791-637e437bbe8e", "doc_id": "d1ed01e5-ec86-4e42-8791-637e437bbe8e", "ref_doc_id": "d1ed01e5-ec86-4e42-8791-637e437bbe8e", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Configuration\n\n[`PeftConfigMixin`] is the base configuration class for storing the adapter configuration of a [`PeftModel`], and [`PromptLearningConfig`] is the base configuration class for soft prompt methods (p-tuning, prefix tuning, and prompt tuning). These base classes contain methods for saving and loading model configurations from the Hub, specifying the PEFT method to use, type of task to perform, and model configurations like number of layers and number of attention heads.\n\n## PeftConfigMixin\n\n[[autodoc]] config.PeftConfigMixin\n    - all\n\n## PeftConfig\n\n[[autodoc]] PeftConfig\n    - all\n\n## PromptLearningConfig\n\n[[autodoc]] PromptLearningConfig\n    - all"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e628a8eb-4b41-4234-ab41-5dacdca34939
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/loha", "title": "LoHa", "tokens": 472, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e628a8eb-4b41-4234-ab41-5dacdca34939\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/loha\", \"title\": \"LoHa\", \"tokens\": 472, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"aa768c6b-a2e9-4361-a245-7b8b702eb037\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/loha\", \"title\": \"LoHa\", \"tokens\": 472, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"e155eb1d0fe0f5ea693f67bf1b4688555076b075ffc6f1b21dacfc4b0b788cdb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1471, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "aa768c6b-a2e9-4361-a245-7b8b702eb037", "doc_id": "aa768c6b-a2e9-4361-a245-7b8b702eb037", "ref_doc_id": "aa768c6b-a2e9-4361-a245-7b8b702eb037", "chroma:document": "# LoHa\n\nLow-Rank Hadamard Product ([LoHa](https://huggingface.co/papers/2108.06098)), is similar to LoRA except it approximates the large weight matrix with more low-rank matrices and combines them with the Hadamard product. This method is even more parameter-efficient than LoRA and achieves comparable performance.\n\nThe abstract from the paper is:\n\n*In this work, we propose a communication-efficient parameterization, FedPara, for federated learning (FL) to overcome the burdens on frequent model uploads and downloads. Our method re-parameterizes weight parameters of layers using low-rank weights followed by the Hadamard product. Compared to the conventional low-rank parameterization, our FedPara method is not restricted to low-rank constraints, and thereby it has a far larger capacity. This property enables to achieve comparable performance while requiring 3 to 10 times lower communication costs than the model with the original layers, which is not achievable by the traditional low-rank methods. The efficiency of our method can be further improved by combining with other efficient FL optimizers. In addition, we extend our method to a personalized FL application, pFedPara, which separates parameters into global and local ones. We show that pFedPara outperforms competing personalized FL methods with more than three times fewer parameters*.\n\n## LoHaConfig\n\n[[autodoc]] tuners.loha.config.LoHaConfig\n\n## LoHaModel\n\n[[autodoc]] tuners.loha.model.LoHaModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: ee5df3bf-ea50-4412-9c84-3cdbcade88a9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/auto_class", "title": "AutoPeftModels", "tokens": 406, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"ee5df3bf-ea50-4412-9c84-3cdbcade88a9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/auto_class\", \"title\": \"AutoPeftModels\", \"tokens\": 406, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0f444c42-694c-432a-8122-e863d92e303c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/auto_class\", \"title\": \"AutoPeftModels\", \"tokens\": 406, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"1687cbcaa2dde64f3e62c79d91e5a386e8895d4c173d4d0ee638e907cff87209\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 931, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0f444c42-694c-432a-8122-e863d92e303c", "doc_id": "0f444c42-694c-432a-8122-e863d92e303c", "ref_doc_id": "0f444c42-694c-432a-8122-e863d92e303c", "chroma:document": "# AutoPeftModels\n\nThe `AutoPeftModel` classes loads the appropriate PEFT model for the task type by automatically inferring it from the configuration file. They are designed to quickly and easily load a PEFT model in a single line of code without having to worry about which exact model class you need or manually loading a [`PeftConfig`].\n\n## AutoPeftModel\n\n[[autodoc]] auto.AutoPeftModel\n    - from_pretrained\n\n## AutoPeftModelForCausalLM\n\n[[autodoc]] auto.AutoPeftModelForCausalLM\n\n## AutoPeftModelForSeq2SeqLM\n\n[[autodoc]] auto.AutoPeftModelForSeq2SeqLM\n\n## AutoPeftModelForSequenceClassification\n\n[[autodoc]] auto.AutoPeftModelForSequenceClassification\n\n## AutoPeftModelForTokenClassification\n\n[[autodoc]] auto.AutoPeftModelForTokenClassification\n\n## AutoPeftModelForQuestionAnswering\n\n[[autodoc]] auto.AutoPeftModelForQuestionAnswering\n\n## AutoPeftModelForFeatureExtraction\n\n[[autodoc]] auto.AutoPeftModelForFeatureExtraction"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 795d32fa-7728-4b2d-aa17-9bd523474b02
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/fourierft", "title": "FourierFT: Discrete Fourier Transformation Fine-Tuning", "tokens": 562, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"795d32fa-7728-4b2d-aa17-9bd523474b02\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/fourierft\", \"title\": \"FourierFT: Discrete Fourier Transformation Fine-Tuning\", \"tokens\": 562, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1d00cdad-4a75-45c9-9295-f82d1cd59b4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/fourierft\", \"title\": \"FourierFT: Discrete Fourier Transformation Fine-Tuning\", \"tokens\": 562, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a27cf076f5a4f11b7dd22c68f92a7b4ee42d09324bf1c7666a52d32c814b0692\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1921, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1d00cdad-4a75-45c9-9295-f82d1cd59b4e", "doc_id": "1d00cdad-4a75-45c9-9295-f82d1cd59b4e", "ref_doc_id": "1d00cdad-4a75-45c9-9295-f82d1cd59b4e", "chroma:document": "# FourierFT: Discrete Fourier Transformation Fine-Tuning\n\n[FourierFT](https://huggingface.co/papers/2405.03003) is a parameter-efficient fine-tuning technique that leverages Discrete Fourier Transform to compress the model's tunable weights. This method outperforms LoRA in the GLUE benchmark and common ViT classification tasks using much less parameters.\n\nFourierFT currently has the following constraints:\n\n- Only `nn.Linear` layers are supported.\n- Quantized layers are not supported.\n\nIf these constraints don't work for your use case, consider other methods instead.\n\nThe abstract from the paper is:\n\n> Low-rank adaptation (LoRA) has recently gained much interest in fine-tuning foundation models. It effectively reduces the number of trainable parameters by incorporating low-rank matrices A and B to represent the weight change, i.e., Delta W=BA. Despite LoRA's progress, it faces storage challenges when handling extensive customization adaptations or larger base models. In this work, we aim to further compress trainable parameters by enjoying the powerful expressiveness of the Fourier transform. Specifically, we introduce FourierFT, which treats Delta W as a matrix in the spatial domain and learns only a small fraction of its spectral coefficients. With the trained spectral coefficients, we implement the inverse discrete Fourier transform to recover Delta W. Empirically, our FourierFT method shows comparable or better performance with fewer parameters than LoRA on various tasks, including natural language understanding, natural language generation, instruction tuning, and image classification. For example, when performing instruction tuning on the LLaMA2-7B model, FourierFT surpasses LoRA with only 0.064M trainable parameters, compared to LoRA's 33.5M.\n\n## FourierFTConfig\n\n[[autodoc]] tuners.fourierft.config.FourierFTConfig\n\n## FourierFTModel\n\n[[autodoc]] tuners.fourierft.model.FourierFTModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: c2c0513c-0446-42f5-a7bb-d52aaf634294
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/p_tuning", "title": "P-tuning", "tokens": 474, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"c2c0513c-0446-42f5-a7bb-d52aaf634294\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/p_tuning\", \"title\": \"P-tuning\", \"tokens\": 474, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"832c1f39-3f27-43e3-a3e1-3aac390e3065\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/p_tuning\", \"title\": \"P-tuning\", \"tokens\": 474, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3da9718298c73aa418d8a1e4527acd249171096a2d00b324c7d6ed91767dc17c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1439, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "832c1f39-3f27-43e3-a3e1-3aac390e3065", "doc_id": "832c1f39-3f27-43e3-a3e1-3aac390e3065", "ref_doc_id": "832c1f39-3f27-43e3-a3e1-3aac390e3065", "chroma:document": "# P-tuning\n\n[P-tuning](https://hf.co/papers/2103.10385) adds trainable prompt embeddings to the input that is optimized by a prompt encoder to find a better prompt, eliminating the need to manually design prompts. The prompt tokens can be added anywhere in the input sequence, and p-tuning also introduces anchor tokens for improving performance.\n\nThe abstract from the paper is:\n\n*While GPTs with traditional fine-tuning fail to achieve strong results on natural language understanding (NLU), we show that GPTs can be better than or comparable to similar-sized BERTs on NLU tasks with a novel method P-tuning -- which employs trainable continuous prompt embeddings. On the knowledge probing (LAMA) benchmark, the best GPT recovers 64\\% (P@1) of world knowledge without any additional text provided during test time, which substantially improves the previous best by 20+ percentage points. On the SuperGlue benchmark, GPTs achieve comparable and sometimes better performance to similar-sized BERTs in supervised learning. Importantly, we find that P-tuning also improves BERTs' performance in both few-shot and supervised settings while largely reducing the need for prompt engineering. Consequently, P-tuning outperforms the state-of-the-art approaches on the few-shot SuperGlue benchmark.*.\n\n## PromptEncoderConfig\n\n[[autodoc]] tuners.p_tuning.config.PromptEncoderConfig\n\n## PromptEncoder\n\n[[autodoc]] tuners.p_tuning.model.PromptEncoder"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: fc0bd42a-771a-4f4a-899e-73f757063ab4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/merge_utils", "title": "Model merge", "tokens": 282, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"fc0bd42a-771a-4f4a-899e-73f757063ab4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/merge_utils\", \"title\": \"Model merge\", \"tokens\": 282, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7431cfef-7481-44c6-aabe-f87da15343e0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/merge_utils\", \"title\": \"Model merge\", \"tokens\": 282, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"24fe8e3962bd13e7b37a630d61392ea5baddd9e790261273beacdf9b5db64f23\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 460, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7431cfef-7481-44c6-aabe-f87da15343e0", "doc_id": "7431cfef-7481-44c6-aabe-f87da15343e0", "ref_doc_id": "7431cfef-7481-44c6-aabe-f87da15343e0", "chroma:document": "# Model merge\n\nPEFT provides several internal utilities for [merging LoRA adapters](../developer_guides/model_merging) with the TIES and DARE methods.\n\n[[autodoc]] utils.merge_utils.prune\n\n[[autodoc]] utils.merge_utils.calculate_majority_sign_mask\n\n[[autodoc]] utils.merge_utils.disjoint_merge\n\n[[autodoc]] utils.merge_utils.task_arithmetic\n\n[[autodoc]] utils.merge_utils.ties\n\n[[autodoc]] utils.merge_utils.dare_linear\n\n[[autodoc]] utils.merge_utils.dare_ties"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 835d875f-dfa4-4df6-9b32-8d1aa7629aae
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/peft_types", "title": "PEFT types", "tokens": 224, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"835d875f-dfa4-4df6-9b32-8d1aa7629aae\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/peft_types\", \"title\": \"PEFT types\", \"tokens\": 224, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a71df622-84fc-4a0e-8cf2-9398f412164c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/peft_types\", \"title\": \"PEFT types\", \"tokens\": 224, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"0a5a2da2339682b70f8078ef3a4b7855c9ce55c13a872abeb7bc60375386bbc9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 219, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a71df622-84fc-4a0e-8cf2-9398f412164c", "doc_id": "a71df622-84fc-4a0e-8cf2-9398f412164c", "ref_doc_id": "a71df622-84fc-4a0e-8cf2-9398f412164c", "chroma:document": "# PEFT types\n\n[`PeftType`] includes the supported adapters in PEFT, and [`TaskType`] includes PEFT-supported tasks.\n\n## PeftType\n\n[[autodoc]] utils.peft_types.PeftType\n\n## TaskType\n\n[[autodoc]] utils.peft_types.TaskType"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e7282b82-2b7f-4b96-b0b4-26eb071153d0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/poly", "title": "Polytropon", "tokens": 976, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e7282b82-2b7f-4b96-b0b4-26eb071153d0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/poly\", \"title\": \"Polytropon\", \"tokens\": 976, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f784c64c-0c8d-42d8-8e72-92f906f6c5ce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/poly\", \"title\": \"Polytropon\", \"tokens\": 976, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"7b847d0f05bd8c611d26aab4e3dc0643911706d7ce8e6bcdaf772ce55343e1c7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8a8f39d3-6a7c-4877-82ad-fa9f831558fc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9aaca8e31a6177bf8cec81c2d4b81eb2019a76ed607e45ae89441d48d44cf8d9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3705, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "doc_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "ref_doc_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "chroma:document": "# Polytropon\n\n[Polytropon](https://hf.co/papers/2202.13914) is a multitask model with a number of different LoRA adapters in it's \"inventory\". The model learns the correct combination of adapters from the inventory with a routing function to choose the best subset of modules for a specific task. PEFT also supports [Multi-Head Adapter Routing (MHR)](https://hf.co/papers/2211.03831) for Polytropon which builds on and improves the routing function by combining the adapter heads more granularly. The adapter heads are separated into disjoint blocks and a different routing function is learned for each one, allowing for more expressivity.\n\n<hfoptions id=\"paper\">\n<hfoption id=\"Combining Modular Skills in Multitask Learning\">\n\nThe abstract from the paper is:\n\n*A modular design encourages neural models to disentangle and recombine different facets of knowledge to generalise more systematically to new tasks. In this work, we assume that each task is associated with a subset of latent discrete skills from a (potentially small) inventory. In turn, skills correspond to parameter-efficient (sparse / low-rank) model parameterisations. By jointly learning these and a task-skill allocation matrix, the network for each task is instantiated as the average of the parameters of active skills. To favour non-trivial soft partitions of skills across tasks, we experiment with a series of inductive biases, such as an Indian Buffet Process prior and a two-speed learning rate. We evaluate our latent-skill model on two main settings: 1) multitask reinforcement learning for grounded instruction following on 8 levels of the BabyAI platform; and 2) few-shot adaptation of pre-trained text-to-text generative models on CrossFit, a benchmark comprising 160 NLP tasks. We find that the modular design of a network significantly increases sample efficiency in reinforcement learning and few-shot generalisation in supervised learning, compared to baselines with fully shared, task-specific, or conditionally generated parameters where knowledge is entangled across tasks. In addition, we show how discrete skills help interpretability, as they yield an explicit hierarchy of tasks.*\n\n</hfoption>\n<hfoption id=\"Multi-Head Adapter Routing for Cross-Task Generalization\">\n\nThe abstract from the paper is:\n\n*Parameter-efficient fine-tuning (PEFT) for cross-task generalization consists in pre-training adapters on a multi-task training set before few-shot adaptation to test tasks. Polytropon [Ponti et al., 2023] (Poly) jointly learns an inventory of adapters and a routing function that selects a (variable-size) subset of adapters for each task during both pre-training and few-shot adaptation. In this paper, we investigate the role that adapter routing plays in its success and design new variants based on our findings. First, we build on the intuition that finer-grained routing provides more expressivity. Hence, we propose MHR (Multi-Head Routing), which combines subsets of adapter parameters and outperforms Poly under a comparable parameter budget; by only fine-tuning the routing function and not the adapters (MHR-z), we achieve competitive performance with extreme parameter efficiency. Second, we find that Poly/MHR performance is a result of better multi-task optimization, rather than modular inductive biases that facilitate adapter recombination and local adaptation, as previously hypothesized. In fact, we find that MHR exhibits higher gradient alignment between tasks than any other method. Since this implies that routing is only crucial during multi-task pre-training, we propose MHR-mu, which discards routing and fine-tunes the average of the pre-trained adapters during few-shot adaptation."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 8a8f39d3-6a7c-4877-82ad-fa9f831558fc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/poly", "title": "Polytropon", "tokens": 976, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"8a8f39d3-6a7c-4877-82ad-fa9f831558fc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/poly\", \"title\": \"Polytropon\", \"tokens\": 976, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f784c64c-0c8d-42d8-8e72-92f906f6c5ce\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/poly\", \"title\": \"Polytropon\", \"tokens\": 976, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"7b847d0f05bd8c611d26aab4e3dc0643911706d7ce8e6bcdaf772ce55343e1c7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e7282b82-2b7f-4b96-b0b4-26eb071153d0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/poly\", \"title\": \"Polytropon\", \"tokens\": 976, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8b356614c969d5dda65e9672e6f7e05a0c4b96a61ee14d8164d7d288f5e92ee1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1761, \"end_char_idx\": 3925, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "doc_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "ref_doc_id": "f784c64c-0c8d-42d8-8e72-92f906f6c5ce", "chroma:document": "We find that the modular design of a network significantly increases sample efficiency in reinforcement learning and few-shot generalisation in supervised learning, compared to baselines with fully shared, task-specific, or conditionally generated parameters where knowledge is entangled across tasks. In addition, we show how discrete skills help interpretability, as they yield an explicit hierarchy of tasks.*\n\n</hfoption>\n<hfoption id=\"Multi-Head Adapter Routing for Cross-Task Generalization\">\n\nThe abstract from the paper is:\n\n*Parameter-efficient fine-tuning (PEFT) for cross-task generalization consists in pre-training adapters on a multi-task training set before few-shot adaptation to test tasks. Polytropon [Ponti et al., 2023] (Poly) jointly learns an inventory of adapters and a routing function that selects a (variable-size) subset of adapters for each task during both pre-training and few-shot adaptation. In this paper, we investigate the role that adapter routing plays in its success and design new variants based on our findings. First, we build on the intuition that finer-grained routing provides more expressivity. Hence, we propose MHR (Multi-Head Routing), which combines subsets of adapter parameters and outperforms Poly under a comparable parameter budget; by only fine-tuning the routing function and not the adapters (MHR-z), we achieve competitive performance with extreme parameter efficiency. Second, we find that Poly/MHR performance is a result of better multi-task optimization, rather than modular inductive biases that facilitate adapter recombination and local adaptation, as previously hypothesized. In fact, we find that MHR exhibits higher gradient alignment between tasks than any other method. Since this implies that routing is only crucial during multi-task pre-training, we propose MHR-mu, which discards routing and fine-tunes the average of the pre-trained adapters during few-shot adaptation. This establishes MHR-mu as an effective method for single-adapter fine-tuning.*.\n\n</hfoption>\n</hfoptions>\n\n## PolyConfig\n\n[[autodoc]] tuners.poly.config.PolyConfig\n\n## PolyModel\n\n[[autodoc]] tuners.poly.model.PolyModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 89962af8-1d6c-4a7c-9a73-a38e16640b6f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/oft", "title": "OFT", "tokens": 522, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"89962af8-1d6c-4a7c-9a73-a38e16640b6f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/oft\", \"title\": \"OFT\", \"tokens\": 522, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7561928c-46f2-4762-a7ee-659a7dd26be6\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/oft\", \"title\": \"OFT\", \"tokens\": 522, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"f3ca16c78fe957ed1a12aeb2ed0f25a9ab34ae9452bb3b2c8dd93366cd836580\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1794, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7561928c-46f2-4762-a7ee-659a7dd26be6", "doc_id": "7561928c-46f2-4762-a7ee-659a7dd26be6", "ref_doc_id": "7561928c-46f2-4762-a7ee-659a7dd26be6", "chroma:document": "# OFT\n\n[Orthogonal Finetuning (OFT)](https://hf.co/papers/2306.07280) is a method developed for adapting text-to-image diffusion models. It works by reparameterizing the pretrained weight matrices with it's orthogonal matrix to preserve information in the pretrained model. To reduce the number of parameters, OFT introduces a block-diagonal structure in the orthogonal matrix.\n\nThe abstract from the paper is:\n\n*Large text-to-image diffusion models have impressive capabilities in generating photorealistic images from text prompts. How to effectively guide or control these powerful models to perform different downstream tasks becomes an important open problem. To tackle this challenge, we introduce a principled finetuning method -- Orthogonal Finetuning (OFT), for adapting text-to-image diffusion models to downstream tasks. Unlike existing methods, OFT can provably preserve hyperspherical energy which characterizes the pairwise neuron relationship on the unit hypersphere. We find that this property is crucial for preserving the semantic generation ability of text-to-image diffusion models. To improve finetuning stability, we further propose Constrained Orthogonal Finetuning (COFT) which imposes an additional radius constraint to the hypersphere. Specifically, we consider two important finetuning text-to-image tasks: subject-driven generation where the goal is to generate subject-specific images given a few images of a subject and a text prompt, and controllable generation where the goal is to enable the model to take in additional control signals. We empirically show that our OFT framework outperforms existing methods in generation quality and convergence speed*.\n\n## OFTConfig\n\n[[autodoc]] tuners.oft.config.OFTConfig\n\n## OFTModel\n\n[[autodoc]] tuners.oft.model.OFTModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 57489d08-5c80-4c36-bbee-a294820f7290
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/adapter_utils", "title": "LyCORIS", "tokens": 318, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"57489d08-5c80-4c36-bbee-a294820f7290\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/adapter_utils\", \"title\": \"LyCORIS\", \"tokens\": 318, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a6b7cc5b-2a76-45dd-be7a-c25e46fd4f84\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/adapter_utils\", \"title\": \"LyCORIS\", \"tokens\": 318, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d2d990522f6dda7c3c52f82c8f2d5a1cd3b4601a59a38b8b0d20cf6d8c164046\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 520, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a6b7cc5b-2a76-45dd-be7a-c25e46fd4f84", "doc_id": "a6b7cc5b-2a76-45dd-be7a-c25e46fd4f84", "ref_doc_id": "a6b7cc5b-2a76-45dd-be7a-c25e46fd4f84", "chroma:document": "# LyCORIS\n\n[LyCORIS](https://hf.co/papers/2309.14859) (Lora beYond Conventional methods, Other Rank adaptation Implementations for Stable diffusion) are LoRA-like matrix decomposition adapters that modify the cross-attention layer of the UNet. The [LoHa](loha) and [LoKr](lokr) methods inherit from the `Lycoris` classes here.\n\n## LycorisConfig\n\n[[autodoc]] tuners.lycoris_utils.LycorisConfig\n\n## LycorisLayer\n\n[[autodoc]] tuners.lycoris_utils.LycorisLayer\n\n## LycorisTuner\n\n[[autodoc]] tuners.lycoris_utils.LycorisTuner"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: cecf6f7b-b9d2-4d02-a03f-9b502ea2f8aa
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/prefix_tuning", "title": "Prefix tuning", "tokens": 449, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"cecf6f7b-b9d2-4d02-a03f-9b502ea2f8aa\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/prefix_tuning\", \"title\": \"Prefix tuning\", \"tokens\": 449, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"72d05abc-666f-4990-ad75-705c3a3368a9\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/prefix_tuning\", \"title\": \"Prefix tuning\", \"tokens\": 449, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"ace4c74a78b9989c51a230ad79c241791a07a89df97ca9e5acf73fd5214bd8e9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1380, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "72d05abc-666f-4990-ad75-705c3a3368a9", "doc_id": "72d05abc-666f-4990-ad75-705c3a3368a9", "ref_doc_id": "72d05abc-666f-4990-ad75-705c3a3368a9", "chroma:document": "# Prefix tuning\n\n[Prefix tuning](https://hf.co/papers/2101.00190) prefixes a series of task-specific vectors to the input sequence that can be learned while keeping the pretrained model frozen. The prefix parameters are inserted in all of the model layers.\n\nThe abstract from the paper is:\n\n*Fine-tuning is the de facto way to leverage large pretrained language models to perform downstream tasks. However, it modifies all the language model parameters and therefore necessitates storing a full copy for each task. In this paper, we propose prefix-tuning, a lightweight alternative to fine-tuning for natural language generation tasks, which keeps language model parameters frozen, but optimizes a small continuous task-specific vector (called the prefix). Prefix-tuning draws inspiration from prompting, allowing subsequent tokens to attend to this prefix as if it were \"virtual tokens\". We apply prefix-tuning to GPT-2 for table-to-text generation and to BART for summarization. We find that by learning only 0.1\\% of the parameters, prefix-tuning obtains comparable performance in the full data setting, outperforms fine-tuning in low-data settings, and extrapolates better to examples with topics unseen during training*.\n\n## PrefixTuningConfig\n\n[[autodoc]] tuners.prefix_tuning.config.PrefixTuningConfig\n\n## PrefixEncoder\n\n[[autodoc]] tuners.prefix_tuning.model.PrefixEncoder"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: f95f207f-f0ef-44e8-850b-e867fb30ea8d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/package_reference/lokr", "title": "LoKr", "tokens": 277, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"f95f207f-f0ef-44e8-850b-e867fb30ea8d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/lokr\", \"title\": \"LoKr\", \"tokens\": 277, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d4ba7d4d-4cb7-4556-8d2a-f5c2990414aa\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/package_reference/lokr\", \"title\": \"LoKr\", \"tokens\": 277, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"ff4b8b326973fece15243daf72021e7ebdc8d074ece8c5043f37e606d1311ec7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 429, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d4ba7d4d-4cb7-4556-8d2a-f5c2990414aa", "doc_id": "d4ba7d4d-4cb7-4556-8d2a-f5c2990414aa", "ref_doc_id": "d4ba7d4d-4cb7-4556-8d2a-f5c2990414aa", "chroma:document": "# LoKr\n\nLow-Rank Kronecker Product ([LoKr](https://hf.co/papers/2309.14859)), is a LoRA-variant method that approximates the large weight matrix with two low-rank matrices and combines them with the Kronecker product. LoKr also provides an optional third low-rank matrix to provide better control during fine-tuning.\n\n## LoKrConfig\n\n[[autodoc]] tuners.lokr.config.LoKrConfig\n\n## LoKrModel\n\n[[autodoc]] tuners.lokr.model.LoKrModel"}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 7a6ec080-7ab5-4a20-993a-e5a24c152625
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/mixed_models", "title": "Mixed adapter types", "tokens": 659, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"7a6ec080-7ab5-4a20-993a-e5a24c152625\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/mixed_models\", \"title\": \"Mixed adapter types\", \"tokens\": 659, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"51be979c-4326-4efa-9edc-1c6cceb6f322\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/mixed_models\", \"title\": \"Mixed adapter types\", \"tokens\": 659, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"2f6ea15f66eea267a75fc037cc258b4ffb10b03d52a4d8c4e15d3a864eb261e8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2190, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "51be979c-4326-4efa-9edc-1c6cceb6f322", "doc_id": "51be979c-4326-4efa-9edc-1c6cceb6f322", "ref_doc_id": "51be979c-4326-4efa-9edc-1c6cceb6f322", "chroma:document": "# Mixed adapter types\n\nNormally, it isn't possible to mix different adapter types in \ud83e\udd17 PEFT. You can create a PEFT model with two different LoRA adapters (which can have different config options), but it is not possible to combine a LoRA and LoHa adapter. With [`PeftMixedModel`] however, this works as long as the adapter types are compatible. The main purpose of allowing mixed adapter types is to combine trained adapters for inference. While it is possible to train a mixed adapter model, this has not been tested and is not recommended.\n\nTo load different adapter types into a PEFT model, use [`PeftMixedModel`] instead of [`PeftModel`]:\n\n```py\nfrom peft import PeftMixedModel\n\nbase_model = ...  # load the base model, e.g. from transformers\n# load first adapter, which will be called \"default\"\npeft_model = PeftMixedModel.from_pretrained(base_model, <path_to_adapter1>)\npeft_model.load_adapter(<path_to_adapter2>, adapter_name=\"other\")\npeft_model.set_adapter([\"default\", \"other\"])\n```\n\nThe [`~PeftMixedModel.set_adapter`] method is necessary to activate both adapters, otherwise only the first adapter would be active. You can keep adding more adapters by calling [`~PeftModel.add_adapter`] repeatedly.\n\n[`PeftMixedModel`] does not support saving and loading mixed adapters. The adapters should already be trained, and loading the model requires a script to be run each time.\n\n## Tips\n\n- Not all adapter types can be combined. See [`peft.tuners.mixed.COMPATIBLE_TUNER_TYPES`](https://github.com/huggingface/peft/blob/1c1c7fdaa6e6abaa53939b865dee1eded82ad032/src/peft/tuners/mixed/model.py#L35) for a list of compatible types. An error will be raised if you try to combine incompatible adapter types.\n- It is possible to mix multiple adapters of the same type which can be useful for combining adapters with very different configs.\n- If you want to combine a lot of different adapters, the most performant way to do it is to consecutively add the same adapter types. For example, add LoRA1, LoRA2, LoHa1, LoHa2 in this order, instead of LoRA1, LoHa1, LoRA2, and LoHa2. While the order can affect the output, there is no inherently *best* order, so it is best to choose the fastest one."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 81c3b84a-2c98-47d1-affb-8d2ac0dfac88
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"81c3b84a-2c98-47d1-affb-8d2ac0dfac88\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d6a04587-48a2-46da-9a8f-e325c99404f4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d2f70d87a7708ecd38ce103e5faa8ee177a5e1a350184d5ce6a11efea0a9fef8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2916, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "# Troubleshooting\n\nIf you encounter any issue when using PEFT, please check the following list of common issues and their solutions.\n\n## Examples don't work\n\nExamples often rely on the most recent package versions, so please ensure they're up-to-date. In particular, check the following package versions:\n\n- `peft`\n- `transformers`\n- `accelerate`\n- `torch`\n\nIn general, you can update the package version by running this command inside your Python environment:\n\n```bash\npython -m pip install -U <package_name>\n```\n\nInstalling PEFT from source is useful for keeping up with the latest developments:\n\n```bash\npython -m pip install git+https://github.com/huggingface/peft\n```\n\n## ValueError: Attempting to unscale FP16 gradients\n\nThis error probably occurred because the model was loaded with `torch_dtype=torch.float16` and then used in an automatic mixed precision (AMP) context, e.g. by setting `fp16=True` in the [`~transformers.Trainer`] class from \ud83e\udd17 Transformers. The reason is that when using AMP, trainable weights should never use fp16. To make this work without loading the whole model in fp32, add the following to your code:\n\n```python\npeft_model = get_peft_model(...)\n\n# add this:\nfor param in model.parameters():\n    if param.requires_grad:\n        param.data = param.data.float()\n\n# proceed as usual\ntrainer = Trainer(model=peft_model, fp16=True, ...)\ntrainer.train()\n```\n\nAlternatively, you can use the [`~utils.cast_mixed_precision_params`] function to correctly cast the weights:\n\n```python\nfrom peft import cast_mixed_precision_params\n\npeft_model = get_peft_model(...)\ncast_mixed_precision_params(peft_model, dtype=torch.float16)\n\n# proceed as usual\ntrainer = Trainer(model=peft_model, fp16=True, ...)\ntrainer.train()\n```\n\n<Tip>\n\nStarting from PEFT verion v0.12.0, PEFT automatically promotes the dtype of adapter weights from `torch.float16` and `torch.bfloat16` to `torch.float32` where appropriate. To _prevent_ this behavior, you can pass `autocast_adapter_dtype=False` to [`~get_peft_model`], to [`~PeftModel.from_pretrained`], and to [`~PeftModel.load_adapter`].\n\n</Tip>\n\n## Bad results from a loaded PEFT model\n\nThere can be several reasons for getting a poor result from a loaded PEFT model which are listed below. If you're still unable to troubleshoot the problem, see if anyone else had a similar [issue](https://github.com/huggingface/peft/issues) on GitHub, and if you can't find any, open a new issue.\n\nWhen opening an issue, it helps a lot if you provide a minimal code example that reproduces the issue. Also, please report if the loaded model performs at the same level as the model did before fine-tuning, if it performs at a random level, or if it is only slightly worse than expected. This information helps us identify the problem more quickly.\n\n### Random deviations\n\nIf your model outputs are not exactly the same as previous runs, there could be an issue with random elements."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d6a04587-48a2-46da-9a8f-e325c99404f4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d6a04587-48a2-46da-9a8f-e325c99404f4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"81c3b84a-2c98-47d1-affb-8d2ac0dfac88\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3524d599c00d8619816ccdac260bf4bcf6ea7870dce2d5301eff87c98c4b055f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0000b29d-8334-4603-9e02-efcd6c8b74d6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5ff0a4b4616f9ecb4905fa70b35345593a473ddfa3c3ebc99ee0359711cdda85\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1918, \"end_char_idx\": 5481, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "To _prevent_ this behavior, you can pass `autocast_adapter_dtype=False` to [`~get_peft_model`], to [`~PeftModel.from_pretrained`], and to [`~PeftModel.load_adapter`].\n\n</Tip>\n\n## Bad results from a loaded PEFT model\n\nThere can be several reasons for getting a poor result from a loaded PEFT model which are listed below. If you're still unable to troubleshoot the problem, see if anyone else had a similar [issue](https://github.com/huggingface/peft/issues) on GitHub, and if you can't find any, open a new issue.\n\nWhen opening an issue, it helps a lot if you provide a minimal code example that reproduces the issue. Also, please report if the loaded model performs at the same level as the model did before fine-tuning, if it performs at a random level, or if it is only slightly worse than expected. This information helps us identify the problem more quickly.\n\n### Random deviations\n\nIf your model outputs are not exactly the same as previous runs, there could be an issue with random elements. For example:\n\n1. please ensure it is in `.eval()` mode, which is important, for instance, if the model uses dropout\n2. if you use [`~transformers.GenerationMixin.generate`] on a language model, there could be random sampling, so obtaining the same result requires setting a random seed\n3. if you used quantization and merged the weights, small deviations are expected due to rounding errors\n\n### Incorrectly loaded model\n\nPlease ensure that you load the model correctly. A common error is trying to load a _trained_ model with [`get_peft_model`] which is incorrect. Instead, the loading code should look like this:\n\n```python\nfrom peft import PeftModel, PeftConfig\n\nbase_model = ...  # to load the base model, use the same code as when you trained it\nconfig = PeftConfig.from_pretrained(peft_model_id)\npeft_model = PeftModel.from_pretrained(base_model, peft_model_id)\n```\n\n### Randomly initialized layers\n\nFor some tasks, it is important to correctly configure `modules_to_save` in the config to account for randomly initialized layers. \n\nAs an example, this is necessary if you use LoRA to fine-tune a language model for sequence classification because \ud83e\udd17 Transformers adds a randomly initialized classification head on top of the model. If you do not add this layer to `modules_to_save`, the classification head won't be saved. The next time you load the model, you'll get a _different_ randomly initialized classification head, resulting in completely different results.\n\nPEFT tries to correctly guess the `modules_to_save` if you provide the `task_type` argument in the config. This should work for transformers models that follow the standard naming scheme. It is always a good idea to double check though because we can't guarantee all models follow the naming scheme.\n\nWhen you load a transformers model that has randomly initialized layers, you should see a warning along the lines of:\n\n```\nSome weights of <MODEL> were not initialized from the model checkpoint at <ID> and are newly initialized: [<LAYER_NAMES>].\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n```\n\nThe mentioned layers should be added to `modules_to_save` in the config to avoid the described problem.\n\n### Extending the vocabulary\n\nFor many language fine-tuning tasks, extending the model's vocabulary is necessary since new tokens are being introduced. This requires extending the embedding layer to account for the new tokens and also storing the embedding layer in addition to the adapter weights when saving the adapter."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 0000b29d-8334-4603-9e02-efcd6c8b74d6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"0000b29d-8334-4603-9e02-efcd6c8b74d6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d6a04587-48a2-46da-9a8f-e325c99404f4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"4d9d5015cda1cef2cd8054457bdc9c5635331510627c1da6181e1ce983cccdcd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d85cdf84-c7fc-4244-8071-62c3582074b9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a1343d5e18881a6c72516cd373b1360e389f827a3ea97bae6bd926483a00ab4d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3956, \"end_char_idx\": 7263, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "As an example, this is necessary if you use LoRA to fine-tune a language model for sequence classification because \ud83e\udd17 Transformers adds a randomly initialized classification head on top of the model. If you do not add this layer to `modules_to_save`, the classification head won't be saved. The next time you load the model, you'll get a _different_ randomly initialized classification head, resulting in completely different results.\n\nPEFT tries to correctly guess the `modules_to_save` if you provide the `task_type` argument in the config. This should work for transformers models that follow the standard naming scheme. It is always a good idea to double check though because we can't guarantee all models follow the naming scheme.\n\nWhen you load a transformers model that has randomly initialized layers, you should see a warning along the lines of:\n\n```\nSome weights of <MODEL> were not initialized from the model checkpoint at <ID> and are newly initialized: [<LAYER_NAMES>].\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n```\n\nThe mentioned layers should be added to `modules_to_save` in the config to avoid the described problem.\n\n### Extending the vocabulary\n\nFor many language fine-tuning tasks, extending the model's vocabulary is necessary since new tokens are being introduced. This requires extending the embedding layer to account for the new tokens and also storing the embedding layer in addition to the adapter weights when saving the adapter.\n\nSave the embedding layer by adding it to the `target_modules` of the config. The embedding layer name must follow the standard naming scheme from Transformers. For example, the Mistral config could look like this:\n\n```python\nconfig = LoraConfig(..., target_modules=[\"embed_tokens\", \"lm_head\", \"q_proj\", \"v_proj\"])\n```\n\nOnce added to `target_modules`, PEFT automatically stores the embedding layer when saving the adapter if the model has the [`~transformers.PreTrainedModel.get_input_embeddings`] and [`~transformers.PreTrainedModel.get_output_embeddings`]. This is generally the case for Transformers models.\n\nIf the model's embedding layer doesn't follow the Transformer's naming scheme, you can still save it by manually passing `save_embedding_layers=True` when saving the adapter:\n\n```python\nmodel = get_peft_model(...)\n# train the model\nmodel.save_pretrained(\"my_adapter\", save_embedding_layers=True)\n```\n\nFor inference, load the base model first and resize it the same way you did before you trained the model. After you've resized the base model, you can load the PEFT checkpoint.\n\nFor a complete example, please check out [this notebook](https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_with_additional_tokens.ipynb).\n\n### Check layer and model status\n\nSometimes a PEFT model can end up in a bad state, especially when handling multiple adapters. There can be some confusion around what adapters exist, which one is active, which one is merged, etc. To help investigate this issue, call the [`~peft.PeftModel.get_layer_status`] and the [`~peft.PeftModel.get_model_status`] methods. \n\nThe [`~peft.PeftModel.get_layer_status`] method gives you a detailed overview of each targeted layer's active, merged, and available adapters."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: d85cdf84-c7fc-4244-8071-62c3582074b9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"d85cdf84-c7fc-4244-8071-62c3582074b9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0000b29d-8334-4603-9e02-efcd6c8b74d6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b54756583eede28361a2cb5696a3fae5bb437e8cb17ffeebdc86f1e428af83fe\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dd9524e3-8220-4e33-9f87-91655bd948e2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7da36c7399678a1f6d4f7bd9ee8dbbf18807a86213969527ef2a483a3f88138d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5483, \"end_char_idx\": 9037, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "Save the embedding layer by adding it to the `target_modules` of the config. The embedding layer name must follow the standard naming scheme from Transformers. For example, the Mistral config could look like this:\n\n```python\nconfig = LoraConfig(..., target_modules=[\"embed_tokens\", \"lm_head\", \"q_proj\", \"v_proj\"])\n```\n\nOnce added to `target_modules`, PEFT automatically stores the embedding layer when saving the adapter if the model has the [`~transformers.PreTrainedModel.get_input_embeddings`] and [`~transformers.PreTrainedModel.get_output_embeddings`]. This is generally the case for Transformers models.\n\nIf the model's embedding layer doesn't follow the Transformer's naming scheme, you can still save it by manually passing `save_embedding_layers=True` when saving the adapter:\n\n```python\nmodel = get_peft_model(...)\n# train the model\nmodel.save_pretrained(\"my_adapter\", save_embedding_layers=True)\n```\n\nFor inference, load the base model first and resize it the same way you did before you trained the model. After you've resized the base model, you can load the PEFT checkpoint.\n\nFor a complete example, please check out [this notebook](https://github.com/huggingface/peft/blob/main/examples/causal_language_modeling/peft_lora_clm_with_additional_tokens.ipynb).\n\n### Check layer and model status\n\nSometimes a PEFT model can end up in a bad state, especially when handling multiple adapters. There can be some confusion around what adapters exist, which one is active, which one is merged, etc. To help investigate this issue, call the [`~peft.PeftModel.get_layer_status`] and the [`~peft.PeftModel.get_model_status`] methods. \n\nThe [`~peft.PeftModel.get_layer_status`] method gives you a detailed overview of each targeted layer's active, merged, and available adapters.\n\n```python\n>>> from transformers import AutoModel\n>>> from peft import get_peft_model, LoraConfig\n\n>>> model_id = \"google/flan-t5-small\"\n>>> model = AutoModel.from_pretrained(model_id)\n>>> model = get_peft_model(model, LoraConfig())\n\n>>> model.get_layer_status()\n[TunerLayerStatus(name='model.encoder.block.0.layer.0.SelfAttention.q',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['default'],\n                  merged_adapters=[],\n                  requires_grad={'default': True},\n                  available_adapters=['default']),\n TunerLayerStatus(name='model.encoder.block.0.layer.0.SelfAttention.v',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['default'],\n                  merged_adapters=[],\n                  requires_grad={'default': True},\n                  available_adapters=['default']),\n...]\n\n>>> model.get_model_status()\nTunerModelStatus(\n    base_model_type='T5Model',\n    adapter_model_type='LoraModel',\n    peft_types={'default': 'LORA'},\n    trainable_params=344064,\n    total_params=60855680,\n    num_adapter_layers=48,\n    enabled=True,\n    active_adapters=['default'],\n    merged_adapters=[],\n    requires_grad={'default': True},\n    available_adapters=['default'],\n)\n```\n\nIn the model state output, you should look out for entries that say `\"irregular\"`. This means PEFT detected an inconsistent state in the model. For instance, if `merged_adapters=\"irregular\"`, it means that for at least one adapter, it was merged on some target modules but not on others. The inference results will most likely be incorrect as a result.\n\nThe best way to resolve this issue is to reload the whole model and adapter checkpoint(s)."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: dd9524e3-8220-4e33-9f87-91655bd948e2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"dd9524e3-8220-4e33-9f87-91655bd948e2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d85cdf84-c7fc-4244-8071-62c3582074b9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a1600899f548bbb8ab6039aae217786354e5ae1b7d31218193c5801f7000fe2f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"46402fb9-3c40-47b3-b2a4-9787f7629e73\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3424987fc654717cf762c13a9d94ebc16589ad173d8a4f48a5824c30bd49a727\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7265, \"end_char_idx\": 9601, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "```python\n>>> from transformers import AutoModel\n>>> from peft import get_peft_model, LoraConfig\n\n>>> model_id = \"google/flan-t5-small\"\n>>> model = AutoModel.from_pretrained(model_id)\n>>> model = get_peft_model(model, LoraConfig())\n\n>>> model.get_layer_status()\n[TunerLayerStatus(name='model.encoder.block.0.layer.0.SelfAttention.q',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['default'],\n                  merged_adapters=[],\n                  requires_grad={'default': True},\n                  available_adapters=['default']),\n TunerLayerStatus(name='model.encoder.block.0.layer.0.SelfAttention.v',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['default'],\n                  merged_adapters=[],\n                  requires_grad={'default': True},\n                  available_adapters=['default']),\n...]\n\n>>> model.get_model_status()\nTunerModelStatus(\n    base_model_type='T5Model',\n    adapter_model_type='LoraModel',\n    peft_types={'default': 'LORA'},\n    trainable_params=344064,\n    total_params=60855680,\n    num_adapter_layers=48,\n    enabled=True,\n    active_adapters=['default'],\n    merged_adapters=[],\n    requires_grad={'default': True},\n    available_adapters=['default'],\n)\n```\n\nIn the model state output, you should look out for entries that say `\"irregular\"`. This means PEFT detected an inconsistent state in the model. For instance, if `merged_adapters=\"irregular\"`, it means that for at least one adapter, it was merged on some target modules but not on others. The inference results will most likely be incorrect as a result.\n\nThe best way to resolve this issue is to reload the whole model and adapter checkpoint(s). Ensure that you don't perform any incorrect operations on the model, e.g. manually merging adapters on some modules but not others.\n\nConvert the layer status into a pandas `DataFrame` for an easier visual inspection.\n\n```python\nfrom dataclasses import asdict\nimport pandas as pd\n\ndf = pd.DataFrame(asdict(layer) for layer in model.get_layer_status())\n```\n\nIt is possible to get this information for non-PEFT models if they are using PEFT layers under the hood, but some information like the `base_model_type` or the `peft_types` cannot be determined in that case."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 46402fb9-3c40-47b3-b2a4-9787f7629e73
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"46402fb9-3c40-47b3-b2a4-9787f7629e73\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dd9524e3-8220-4e33-9f87-91655bd948e2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"3785f197eef5a5f61973f505dd61c19a46976c1c70bfe91650d52b37e96c830c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e5d5e12f-35df-4985-9fb4-6a88a48947d6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4df93fc0b71268c03ade5b4083757a21464b8247819fd33c7277bf4a9c8b10bb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8676, \"end_char_idx\": 11925, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "This means PEFT detected an inconsistent state in the model. For instance, if `merged_adapters=\"irregular\"`, it means that for at least one adapter, it was merged on some target modules but not on others. The inference results will most likely be incorrect as a result.\n\nThe best way to resolve this issue is to reload the whole model and adapter checkpoint(s). Ensure that you don't perform any incorrect operations on the model, e.g. manually merging adapters on some modules but not others.\n\nConvert the layer status into a pandas `DataFrame` for an easier visual inspection.\n\n```python\nfrom dataclasses import asdict\nimport pandas as pd\n\ndf = pd.DataFrame(asdict(layer) for layer in model.get_layer_status())\n```\n\nIt is possible to get this information for non-PEFT models if they are using PEFT layers under the hood, but some information like the `base_model_type` or the `peft_types` cannot be determined in that case. As an example, you can call this on a [diffusers](https://huggingface.co/docs/diffusers/index) model like so:\n\n```python\n>>> import torch\n>>> from diffusers import StableDiffusionPipeline\n>>> from peft import get_model_status, get_layer_status\n\n>>> path = \"runwayml/stable-diffusion-v1-5\"\n>>> lora_id = \"takuma104/lora-test-text-encoder-lora-target\"\n>>> pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16)\n>>> pipe.load_lora_weights(lora_id, adapter_name=\"adapter-1\")\n>>> pipe.load_lora_weights(lora_id, adapter_name=\"adapter-2\")\n>>> pipe.set_lora_device([\"adapter-2\"], \"cuda\")\n>>> get_layer_status(pipe.text_encoder)\n[TunerLayerStatus(name='text_model.encoder.layers.0.self_attn.k_proj',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['adapter-2'],\n                  merged_adapters=[],\n                  requires_grad={'adapter-1': False, 'adapter-2': True},\n                  available_adapters=['adapter-1', 'adapter-2'],\n                  devices={'adapter-1': ['cpu'], 'adapter-2': ['cuda']}),\n TunerLayerStatus(name='text_model.encoder.layers.0.self_attn.v_proj',\n                  module_type='lora.Linear',\n                  enabled=True,\n                  active_adapters=['adapter-2'],\n                  merged_adapters=[],\n                  requires_grad={'adapter-1': False, 'adapter-2': True},\n                  devices={'adapter-1': ['cpu'], 'adapter-2': ['cuda']}),\n...]\n\n>>> get_model_status(pipe.unet)\nTunerModelStatus(\n    base_model_type='other',\n    adapter_model_type='None',\n    peft_types={},\n    trainable_params=797184,\n    total_params=861115332,\n    num_adapter_layers=128,\n    enabled=True,\n    active_adapters=['adapter-2'],\n    merged_adapters=[],\n    requires_grad={'adapter-1': False, 'adapter-2': True},\n    available_adapters=['adapter-1', 'adapter-2'],\n    devices={'adapter-1': ['cpu'], 'adapter-2': ['cuda']},\n)\n```\n\n## Reproducibility\n\n### Models using batch norm\n\nWhen loading a trained PEFT model where the base model uses batch norm (e.g. `torch.nn.BatchNorm1d` or `torch.nn.BatchNorm2d`), you may find that you cannot reproduce the exact same outputs. This is because the batch norm layers keep track of running stats during training, but these stats are not part of the PEFT checkpoint."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: e5d5e12f-35df-4985-9fb4-6a88a48947d6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/troubleshooting", "title": "Troubleshooting", "tokens": 3077, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"e5d5e12f-35df-4985-9fb4-6a88a48947d6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"45770239-4ed4-4b4d-991f-f00a2ccc1b90\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8395d6dfa6752383b9872f6ee5643171ddd149a81b5cc070ca4a103a9c4abcc9\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"46402fb9-3c40-47b3-b2a4-9787f7629e73\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/troubleshooting\", \"title\": \"Troubleshooting\", \"tokens\": 3077, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"d626626c7e7b8f1185897e526aef02cda06a6c0eac90d230b590519a605dd5d5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11676, \"end_char_idx\": 13075, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "ref_doc_id": "45770239-4ed4-4b4d-991f-f00a2ccc1b90", "chroma:document": "`torch.nn.BatchNorm1d` or `torch.nn.BatchNorm2d`), you may find that you cannot reproduce the exact same outputs. This is because the batch norm layers keep track of running stats during training, but these stats are not part of the PEFT checkpoint. Therefore, when you load the PEFT model, the running stats of the base model will be used (i.e. from before training with PEFT).\n\nDepending on your use case, this may not be a big deal. If, however, you need your outputs to be 100% reproducible, you can achieve this by adding the batch norm layers to `modules_to_save`. Below is an example of this using resnet and LoRA. Notice that we set `modules_to_save=[\"classifier\", \"normalization\"]`. We need the `\"classifier\"` argument because our task is image classification, and we add the `\"normalization\"` argument to ensure that the batch norm layers are saved in the PEFT checkpoint.\n\n```python\nfrom transformers import AutoModelForImageClassification\nfrom peft import LoraConfig, get_peft_model\n\nmodel_id = \"microsoft/resnet-18\"\nbase_model = AutoModelForImageClassification.from_pretrained(self.model_id)\nconfig = LoraConfig(\n    target_modules=[\"convolution\"],\n    modules_to_save=[\"classifier\", \"normalization\"],\n),\n```\n\nDepending on the type of model you use, the batch norm layers could have different names than `\"normalization\"`, so please ensure that the name matches your model architecture."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 7554a84d-b0d0-4719-8086-8311c0a2b19f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"7554a84d-b0d0-4719-8086-8311c0a2b19f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ee7a748b-fbee-4d4a-865a-abd0c76a3573\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c36081093ff8a72dbdb18eb6f37dadbdf56682b69837e87f89880cffe61c8656\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3319, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "# LoRA\n\nLoRA is low-rank decomposition method to reduce the number of trainable parameters which speeds up finetuning large models and uses less memory. In PEFT, using LoRA is as easy as setting up a [`LoraConfig`] and wrapping it with [`get_peft_model`] to create a trainable [`PeftModel`].\n\nThis guide explores in more detail other options and features for using LoRA.\n\n## Initialization\n\nThe initialization of LoRA weights is controlled by the parameter `init_lora_weights` in [`LoraConfig`]. By default, PEFT initializes LoRA weights with Kaiming-uniform for weight A and zeros for weight B resulting in an identity transform (same as the reference [implementation](https://github.com/microsoft/LoRA)).\n\nIt is also possible to pass `init_lora_weights=\"gaussian\"`. As the name suggests, this initializes weight A with a Gaussian distribution and zeros for weight B (this is how [Diffusers](https://huggingface.co/docs/diffusers/index) initializes LoRA weights).\n\n```py\nfrom peft import LoraConfig\n\nconfig = LoraConfig(init_lora_weights=\"gaussian\", ...)\n```\n\nThere is also an option to set `init_lora_weights=False` which is useful for debugging and testing. This should be the only time you use this option. When choosing this option, the LoRA weights are initialized such that they do *not* result in an identity transform.\n\n```py\nfrom peft import LoraConfig\n\nconfig = LoraConfig(init_lora_weights=False, ...)\n```\n\n### PiSSA\n[PiSSA](https://arxiv.org/abs/2404.02948) initializes the LoRA adapter using the principal singular values and singular vectors. This straightforward modification allows PiSSA to converge more rapidly than LoRA and ultimately attain superior performance. Moreover, PiSSA reduces the quantization error compared to QLoRA, leading to further enhancements. \n\nConfigure the initialization method to \"pissa\", which may take several minutes to execute SVD on the pre-trained model:\n```python\nfrom peft import LoraConfig\nconfig = LoraConfig(init_lora_weights=\"pissa\", ...)\n```\nAlternatively, execute fast SVD, which takes only a few seconds. The number of iterations determines the trade-off between the error and computation time:\n```python\nlora_config = LoraConfig(init_lora_weights=\"pissa_niter_[number of iters]\", ...) \n```\nFor detailed instruction on using PiSSA, please follow [these instructions](https://github.com/fxmeng/peft/tree/main/examples/pissa_finetuning).\n\n### OLoRA\n[OLoRA](https://arxiv.org/abs/2406.01775) utilizes QR decomposition to initialize the LoRA adapters. OLoRA translates the base weights of the model by a factor of their QR decompositions, i.e., it mutates the weights before performing any training on them. This approach significantly improves stability, accelerates convergence speed, and ultimately achieves superior performance.\n\nYou just need to pass a single additional option to use OLoRA:\n```python\nfrom peft import LoraConfig\nconfig = LoraConfig(init_lora_weights=\"olora\", ...)\n```\nFor more advanced usage, please refer to our [documentation](https://github.com/huggingface/peft/tree/main/examples/olora_finetuning).\n### LoftQ\n\n#### Standard approach\n\nWhen quantizing the base model for QLoRA training, consider using the [LoftQ initialization](https://arxiv.org/abs/2310.08659), which has been shown to improve performance when training quantized models."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: ee7a748b-fbee-4d4a-865a-abd0c76a3573
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"ee7a748b-fbee-4d4a-865a-abd0c76a3573\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7554a84d-b0d0-4719-8086-8311c0a2b19f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"091347ed78d4048ed04daee7ed95dedf3cb02cf4623dd4d9a4962ba30c649599\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7c53a307-1b8d-41d6-b515-faac29ce18fa\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3ab9e82ef5163177701facc254420b97a866440eca3f67d64b10d9f3a1e7ccfd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1684, \"end_char_idx\": 4902, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "Moreover, PiSSA reduces the quantization error compared to QLoRA, leading to further enhancements. \n\nConfigure the initialization method to \"pissa\", which may take several minutes to execute SVD on the pre-trained model:\n```python\nfrom peft import LoraConfig\nconfig = LoraConfig(init_lora_weights=\"pissa\", ...)\n```\nAlternatively, execute fast SVD, which takes only a few seconds. The number of iterations determines the trade-off between the error and computation time:\n```python\nlora_config = LoraConfig(init_lora_weights=\"pissa_niter_[number of iters]\", ...) \n```\nFor detailed instruction on using PiSSA, please follow [these instructions](https://github.com/fxmeng/peft/tree/main/examples/pissa_finetuning).\n\n### OLoRA\n[OLoRA](https://arxiv.org/abs/2406.01775) utilizes QR decomposition to initialize the LoRA adapters. OLoRA translates the base weights of the model by a factor of their QR decompositions, i.e., it mutates the weights before performing any training on them. This approach significantly improves stability, accelerates convergence speed, and ultimately achieves superior performance.\n\nYou just need to pass a single additional option to use OLoRA:\n```python\nfrom peft import LoraConfig\nconfig = LoraConfig(init_lora_weights=\"olora\", ...)\n```\nFor more advanced usage, please refer to our [documentation](https://github.com/huggingface/peft/tree/main/examples/olora_finetuning).\n### LoftQ\n\n#### Standard approach\n\nWhen quantizing the base model for QLoRA training, consider using the [LoftQ initialization](https://arxiv.org/abs/2310.08659), which has been shown to improve performance when training quantized models. The idea is that the LoRA weights are initialized such that the quantization error is minimized. To use LoftQ, follow [these instructions](https://github.com/huggingface/peft/tree/main/examples/loftq_finetuning).\n\nIn general, for LoftQ to work best, it is recommended to target as many layers with LoRA as possible, since those not targeted cannot have LoftQ applied. This means that passing `LoraConfig(..., target_modules=\"all-linear\")` will most likely give the best results. Also, you should use `nf4` as quant type in your quantization config when using 4bit quantization, i.e. `BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type=\"nf4\")`.\n\n#### A more convenient way\n\nAn easier but more limited way to apply LoftQ initialization is to use the convenience function `replace_lora_weights_loftq`. This takes the quantized PEFT model as input and replaces the LoRA weights in-place with their LoftQ-initialized counterparts.\n\n```python\nfrom peft import replace_lora_weights_loftq\nfrom transformers import BitsAndBytesConfig\n\nbnb_config = BitsAndBytesConfig(load_in_4bit=True, ...)\nbase_model = AutoModelForCausalLM.from_pretrained(..., quantization_config=bnb_config)\n# note: don't pass init_lora_weights=\"loftq\" or loftq_config!\nlora_config = LoraConfig(task_type=\"CAUSAL_LM\")\npeft_model = get_peft_model(base_model, lora_config)\nreplace_lora_weights_loftq(peft_model)\n```\n\n`replace_lora_weights_loftq` also allows you to pass a `callback` argument to give you more control over which layers should be modified or not, which empirically can improve the results quite a lot."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 7c53a307-1b8d-41d6-b515-faac29ce18fa
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"7c53a307-1b8d-41d6-b515-faac29ce18fa\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ee7a748b-fbee-4d4a-865a-abd0c76a3573\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"a81020abf730620d0a1c1fcb77bba135a5011d1ebd58ad1d346e5cd3cdfdf616\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"025c9866-e588-42ca-9ea5-f808aafb7769\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"adf06b807ebd7f61b66763421a4c0c8b264966a55fda4a10136a7f9948645655\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3320, \"end_char_idx\": 6354, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "The idea is that the LoRA weights are initialized such that the quantization error is minimized. To use LoftQ, follow [these instructions](https://github.com/huggingface/peft/tree/main/examples/loftq_finetuning).\n\nIn general, for LoftQ to work best, it is recommended to target as many layers with LoRA as possible, since those not targeted cannot have LoftQ applied. This means that passing `LoraConfig(..., target_modules=\"all-linear\")` will most likely give the best results. Also, you should use `nf4` as quant type in your quantization config when using 4bit quantization, i.e. `BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type=\"nf4\")`.\n\n#### A more convenient way\n\nAn easier but more limited way to apply LoftQ initialization is to use the convenience function `replace_lora_weights_loftq`. This takes the quantized PEFT model as input and replaces the LoRA weights in-place with their LoftQ-initialized counterparts.\n\n```python\nfrom peft import replace_lora_weights_loftq\nfrom transformers import BitsAndBytesConfig\n\nbnb_config = BitsAndBytesConfig(load_in_4bit=True, ...)\nbase_model = AutoModelForCausalLM.from_pretrained(..., quantization_config=bnb_config)\n# note: don't pass init_lora_weights=\"loftq\" or loftq_config!\nlora_config = LoraConfig(task_type=\"CAUSAL_LM\")\npeft_model = get_peft_model(base_model, lora_config)\nreplace_lora_weights_loftq(peft_model)\n```\n\n`replace_lora_weights_loftq` also allows you to pass a `callback` argument to give you more control over which layers should be modified or not, which empirically can improve the results quite a lot. To see a more elaborate example of this, check out [this notebook](https://github.com/huggingface/peft/blob/main/examples/loftq_finetuning/LoftQ_weight_replacement.ipynb).\n\n`replace_lora_weights_loftq` implements only one iteration step of LoftQ. This means that only the LoRA weights are updated, instead of iteratevily updating LoRA weights and quantized base model weights. This may lead to lower performance but has the advantage that we can use the original quantized weights derived from the base model, instead of having to keep an extra copy of modified quantized weights. Whether this tradeoff is worthwhile depends on the use case.\n\nAt the moment, `replace_lora_weights_loftq` has these additional limitations:\n\n- Model files must be stored as a `safetensors` file.\n- Only bitsandbytes 4bit quantization is supported.\n\n<Tip>\n\nLearn more about how PEFT works with quantization in the [Quantization](quantization) guide.\n\n</Tip>\n\n### Rank-stabilized LoRA\n\nAnother way to initialize [`LoraConfig`] is with the [rank-stabilized LoRA (rsLoRA)](https://huggingface.co/papers/2312.03732) method. The LoRA architecture scales each adapter during every forward pass by a fixed scalar which is set at initialization and depends on the rank `r`. The scalar is given by `lora_alpha/r` in the original implementation, but rsLoRA uses `lora_alpha/math.sqrt(r)` which stabilizes the adapters and increases the performance potential from using a higher `r`."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 025c9866-e588-42ca-9ea5-f808aafb7769
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"025c9866-e588-42ca-9ea5-f808aafb7769\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7c53a307-1b8d-41d6-b515-faac29ce18fa\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"9eca8d0378c7d3b7387bc5228816b8590cfb9ef3195e9607353dda4269f3e727\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b093821e-e262-47cd-ad15-151c7055725e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"547a06549ad8aa6c502010ce3b47415a8e3993a0c56b4f15f2c65d92e650e8aa\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4903, \"end_char_idx\": 8078, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "To see a more elaborate example of this, check out [this notebook](https://github.com/huggingface/peft/blob/main/examples/loftq_finetuning/LoftQ_weight_replacement.ipynb).\n\n`replace_lora_weights_loftq` implements only one iteration step of LoftQ. This means that only the LoRA weights are updated, instead of iteratevily updating LoRA weights and quantized base model weights. This may lead to lower performance but has the advantage that we can use the original quantized weights derived from the base model, instead of having to keep an extra copy of modified quantized weights. Whether this tradeoff is worthwhile depends on the use case.\n\nAt the moment, `replace_lora_weights_loftq` has these additional limitations:\n\n- Model files must be stored as a `safetensors` file.\n- Only bitsandbytes 4bit quantization is supported.\n\n<Tip>\n\nLearn more about how PEFT works with quantization in the [Quantization](quantization) guide.\n\n</Tip>\n\n### Rank-stabilized LoRA\n\nAnother way to initialize [`LoraConfig`] is with the [rank-stabilized LoRA (rsLoRA)](https://huggingface.co/papers/2312.03732) method. The LoRA architecture scales each adapter during every forward pass by a fixed scalar which is set at initialization and depends on the rank `r`. The scalar is given by `lora_alpha/r` in the original implementation, but rsLoRA uses `lora_alpha/math.sqrt(r)` which stabilizes the adapters and increases the performance potential from using a higher `r`.\n\n```py\nfrom peft import LoraConfig\n\nconfig = LoraConfig(use_rslora=True, ...)\n```\n\n### Weight-Decomposed Low-Rank Adaptation (DoRA)\n\nThis technique decomposes the updates of the weights into two parts, magnitude and direction. Direction is handled by normal LoRA, whereas the magnitude is handled by a separate learnable parameter. This can improve the performance of LoRA, especially at low ranks. For more information on DoRA, see  https://arxiv.org/abs/2402.09353.\n\n```py\nfrom peft import LoraConfig\n\nconfig = LoraConfig(use_dora=True, ...)\n```\n\nIf parts of the model or the DoRA adapter are offloaded to CPU you can get a significant speedup at the cost of some temporary (ephemeral) VRAM overhead by using `ephemeral_gpu_offload=True` in `config.runtime_config`.\n\n```py\nfrom peft import LoraConfig, LoraRuntimeConfig\n\nconfig = LoraConfig(use_dora=True, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=True), ...)\n```\n\nA `PeftModel` with a DoRA adapter can also be loaded with `ephemeral_gpu_offload=True` flag using the `from_pretrained` method as well as the `load_adapter` method.\n\n```py\nfrom peft import PeftModel\n\nmodel = PeftModel.from_pretrained(base_model, peft_model_id, ephemeral_gpu_offload=True)\n```\n\n#### Caveats\n\n- DoRA only supports linear and Conv2d layers at the moment.\n- DoRA introduces a bigger overhead than pure LoRA, so it is recommended to merge weights for inference, see [`LoraModel.merge_and_unload`]. \n- DoRA should work with weights quantized with bitsandbytes (\"QDoRA\"). However, issues have been reported when using QDoRA with DeepSpeed Zero2.\n\n### QLoRA-style training\n\nThe default LoRA settings in PEFT add trainable weights to the query and value layers of each attention block."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: b093821e-e262-47cd-ad15-151c7055725e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"b093821e-e262-47cd-ad15-151c7055725e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"025c9866-e588-42ca-9ea5-f808aafb7769\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"e1d7337699b05fe1145df97905b8253299b61d1036af8f6f9d9fd5729095d63c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2463a46c-d060-4f54-a612-9194720b08ae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1ba63942d93e051e0fad8d438bfd2544569f7c0008dbbcd9c3877dac60c3ac15\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6582, \"end_char_idx\": 9710, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "Direction is handled by normal LoRA, whereas the magnitude is handled by a separate learnable parameter. This can improve the performance of LoRA, especially at low ranks. For more information on DoRA, see  https://arxiv.org/abs/2402.09353.\n\n```py\nfrom peft import LoraConfig\n\nconfig = LoraConfig(use_dora=True, ...)\n```\n\nIf parts of the model or the DoRA adapter are offloaded to CPU you can get a significant speedup at the cost of some temporary (ephemeral) VRAM overhead by using `ephemeral_gpu_offload=True` in `config.runtime_config`.\n\n```py\nfrom peft import LoraConfig, LoraRuntimeConfig\n\nconfig = LoraConfig(use_dora=True, runtime_config=LoraRuntimeConfig(ephemeral_gpu_offload=True), ...)\n```\n\nA `PeftModel` with a DoRA adapter can also be loaded with `ephemeral_gpu_offload=True` flag using the `from_pretrained` method as well as the `load_adapter` method.\n\n```py\nfrom peft import PeftModel\n\nmodel = PeftModel.from_pretrained(base_model, peft_model_id, ephemeral_gpu_offload=True)\n```\n\n#### Caveats\n\n- DoRA only supports linear and Conv2d layers at the moment.\n- DoRA introduces a bigger overhead than pure LoRA, so it is recommended to merge weights for inference, see [`LoraModel.merge_and_unload`]. \n- DoRA should work with weights quantized with bitsandbytes (\"QDoRA\"). However, issues have been reported when using QDoRA with DeepSpeed Zero2.\n\n### QLoRA-style training\n\nThe default LoRA settings in PEFT add trainable weights to the query and value layers of each attention block. But [QLoRA](https://hf.co/papers/2305.14314), which adds trainable weights to all the linear layers of a transformer model, can provide performance equal to a fully finetuned model. To apply LoRA to all the linear layers, like in QLoRA, set `target_modules=\"all-linear\"` (easier than specifying individual modules by name which can vary depending on the architecture).\n\n```py\nconfig = LoraConfig(target_modules=\"all-linear\", ...)\n```\n\n### Memory efficient Layer Replication with LoRA\n\nAn approach used to improve the performance of models is to expand a model by duplicating layers in the model to build a larger model from a pretrained model of a given size. For example increasing a 7B model to a 10B model as described in the [SOLAR](https://arxiv.org/abs/2312.15166) paper. PEFT LoRA supports this kind of expansion in a memory efficient manner that supports further fine-tuning using LoRA adapters attached to the layers post replication of the layers. The replicated layers do not take additional memory as they share the underlying weights so the only additional memory required is the memory for the adapter weights. To use this feature you would create a config with the `layer_replication` argument.\n\n```py\nconfig = LoraConfig(layer_replication=[[0,4], [2,5]], ...)\n```\n\nAssuming the original model had 5 layers `[0, 1, 2 ,3, 4]`, this would create a model with 7 layers arranged as `[0, 1, 2, 3, 2, 3, 4]`. This follows the [mergekit](https://github.com/arcee-ai/mergekit) pass through merge convention where sequences of layers specified as start inclusive and end exclusive tuples are stacked to build the final model."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 2463a46c-d060-4f54-a612-9194720b08ae
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"2463a46c-d060-4f54-a612-9194720b08ae\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b093821e-e262-47cd-ad15-151c7055725e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"258f951688db817ece76015ef3b4a86c19f850001e26cd9ad686ed738c3719c5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"505b828f-7622-4304-84cb-0045784f3940\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1813dbbb29b872108e8cf0b8f2c857edc5dc4134c40558b33d23a24692de0ac5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8079, \"end_char_idx\": 10558, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "But [QLoRA](https://hf.co/papers/2305.14314), which adds trainable weights to all the linear layers of a transformer model, can provide performance equal to a fully finetuned model. To apply LoRA to all the linear layers, like in QLoRA, set `target_modules=\"all-linear\"` (easier than specifying individual modules by name which can vary depending on the architecture).\n\n```py\nconfig = LoraConfig(target_modules=\"all-linear\", ...)\n```\n\n### Memory efficient Layer Replication with LoRA\n\nAn approach used to improve the performance of models is to expand a model by duplicating layers in the model to build a larger model from a pretrained model of a given size. For example increasing a 7B model to a 10B model as described in the [SOLAR](https://arxiv.org/abs/2312.15166) paper. PEFT LoRA supports this kind of expansion in a memory efficient manner that supports further fine-tuning using LoRA adapters attached to the layers post replication of the layers. The replicated layers do not take additional memory as they share the underlying weights so the only additional memory required is the memory for the adapter weights. To use this feature you would create a config with the `layer_replication` argument.\n\n```py\nconfig = LoraConfig(layer_replication=[[0,4], [2,5]], ...)\n```\n\nAssuming the original model had 5 layers `[0, 1, 2 ,3, 4]`, this would create a model with 7 layers arranged as `[0, 1, 2, 3, 2, 3, 4]`. This follows the [mergekit](https://github.com/arcee-ai/mergekit) pass through merge convention where sequences of layers specified as start inclusive and end exclusive tuples are stacked to build the final model. Each layer in the final model gets its own distinct set of LoRA adapters.\n\n[Fewshot-Metamath-OrcaVicuna-Mistral-10B](https://huggingface.co/abacusai/Fewshot-Metamath-OrcaVicuna-Mistral-10B) is an example of a model trained using this method on Mistral-7B expanded to 10B. The\n[adapter_config.json](https://huggingface.co/abacusai/Fewshot-Metamath-OrcaVicuna-Mistral-10B/blob/main/adapter_config.json) shows a sample LoRA adapter config applying this method for fine-tuning.\n\n## Optimizers\n\nLoRA training can optionally include special purpose optimizers. Currently the only such optimizer is LoRA+.\n\n### LoRA+ optimized LoRA\n\nLoRA training can be optimized using [LoRA+](https://arxiv.org/abs/2402.12354), which uses different learning rates for the adapter matrices A and B, shown to increase finetuning speed by up to 2x and performance by 1-2%."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 505b828f-7622-4304-84cb-0045784f3940
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"505b828f-7622-4304-84cb-0045784f3940\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2463a46c-d060-4f54-a612-9194720b08ae\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"fcd5a034eca24c9ed72356e1a16543f89937f0c552b8c80bafa517d6193ac262\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"92d4f8d5-7cc9-40f8-90c0-26b645da868c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ebe80864749b622188c74addb060fea3c5a4061b51ed5cf56a2f9b526ba84c7c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9204, \"end_char_idx\": 11824, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "To use this feature you would create a config with the `layer_replication` argument.\n\n```py\nconfig = LoraConfig(layer_replication=[[0,4], [2,5]], ...)\n```\n\nAssuming the original model had 5 layers `[0, 1, 2 ,3, 4]`, this would create a model with 7 layers arranged as `[0, 1, 2, 3, 2, 3, 4]`. This follows the [mergekit](https://github.com/arcee-ai/mergekit) pass through merge convention where sequences of layers specified as start inclusive and end exclusive tuples are stacked to build the final model. Each layer in the final model gets its own distinct set of LoRA adapters.\n\n[Fewshot-Metamath-OrcaVicuna-Mistral-10B](https://huggingface.co/abacusai/Fewshot-Metamath-OrcaVicuna-Mistral-10B) is an example of a model trained using this method on Mistral-7B expanded to 10B. The\n[adapter_config.json](https://huggingface.co/abacusai/Fewshot-Metamath-OrcaVicuna-Mistral-10B/blob/main/adapter_config.json) shows a sample LoRA adapter config applying this method for fine-tuning.\n\n## Optimizers\n\nLoRA training can optionally include special purpose optimizers. Currently the only such optimizer is LoRA+.\n\n### LoRA+ optimized LoRA\n\nLoRA training can be optimized using [LoRA+](https://arxiv.org/abs/2402.12354), which uses different learning rates for the adapter matrices A and B, shown to increase finetuning speed by up to 2x and performance by 1-2%.\n\n```py\nfrom peft import LoraConfig, get_peft_model\nfrom peft.optimizers import create_loraplus_optimizer\nfrom transformers import Trainer\nimport bitsandbytes as bnb\n\nbase_model = ...\nconfig = LoraConfig(...)\nmodel = get_peft_model(base_model, config)\n\noptimizer = create_loraplus_optimizer(\n    model=model,\n    optimizer_cls=bnb.optim.Adam8bit,\n    lr=5e-5,\n    loraplus_lr_ratio=16,\n)\nscheduler = None\n\n...\ntrainer = Trainer(\n    ...,\n    optimizers=(optimizer, scheduler),\n)\n```\n\n## Merge LoRA weights into the base model\n\nWhile LoRA is significantly smaller and faster to train, you may encounter latency issues during inference due to separately loading the base model and the LoRA adapter. To eliminate latency, use the [`~LoraModel.merge_and_unload`] function to merge the adapter weights with the base model. This allows you to use the newly merged model as a standalone model. The [`~LoraModel.merge_and_unload`] function doesn't keep the adapter weights in memory.\n\nBelow is a diagram that explains the intuition of LoRA adapter merging:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora_diagram.png\"/>\n</div>\n\nWe show in the snippets below how to run that using PEFT."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 92d4f8d5-7cc9-40f8-90c0-26b645da868c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"92d4f8d5-7cc9-40f8-90c0-26b645da868c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"505b828f-7622-4304-84cb-0045784f3940\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"8e94d816bd5c168d29672b89bf3c7edbba17bbcdae7ce8bab30a9fc3bdccea5e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"977237cb-de61-4243-8e62-0f7ef6a88813\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"54c009eff3815d7a94466f546412fecf5b44ea757849f62aad994c9a252e2f0b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10186, \"end_char_idx\": 12981, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "## Optimizers\n\nLoRA training can optionally include special purpose optimizers. Currently the only such optimizer is LoRA+.\n\n### LoRA+ optimized LoRA\n\nLoRA training can be optimized using [LoRA+](https://arxiv.org/abs/2402.12354), which uses different learning rates for the adapter matrices A and B, shown to increase finetuning speed by up to 2x and performance by 1-2%.\n\n```py\nfrom peft import LoraConfig, get_peft_model\nfrom peft.optimizers import create_loraplus_optimizer\nfrom transformers import Trainer\nimport bitsandbytes as bnb\n\nbase_model = ...\nconfig = LoraConfig(...)\nmodel = get_peft_model(base_model, config)\n\noptimizer = create_loraplus_optimizer(\n    model=model,\n    optimizer_cls=bnb.optim.Adam8bit,\n    lr=5e-5,\n    loraplus_lr_ratio=16,\n)\nscheduler = None\n\n...\ntrainer = Trainer(\n    ...,\n    optimizers=(optimizer, scheduler),\n)\n```\n\n## Merge LoRA weights into the base model\n\nWhile LoRA is significantly smaller and faster to train, you may encounter latency issues during inference due to separately loading the base model and the LoRA adapter. To eliminate latency, use the [`~LoraModel.merge_and_unload`] function to merge the adapter weights with the base model. This allows you to use the newly merged model as a standalone model. The [`~LoraModel.merge_and_unload`] function doesn't keep the adapter weights in memory.\n\nBelow is a diagram that explains the intuition of LoRA adapter merging:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora_diagram.png\"/>\n</div>\n\nWe show in the snippets below how to run that using PEFT.\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import PeftModel\n\nbase_model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\npeft_model_id = \"alignment-handbook/zephyr-7b-sft-lora\"\nmodel = PeftModel.from_pretrained(base_model, peft_model_id)\nmodel.merge_and_unload()\n```\n\nIf you need to keep a copy of the weights so you can unmerge the adapter later or delete and load different ones, you should use the [`~LoraModel.merge_adapter`] function instead. Now you have the option to use [`~LoraModel.unmerge_adapter`] to return the base model.\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import PeftModel\n\nbase_model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\npeft_model_id = \"alignment-handbook/zephyr-7b-sft-lora\"\nmodel = PeftModel.from_pretrained(base_model, peft_model_id)\nmodel.merge_adapter()\n\n# unmerge the LoRA layers from the base model\nmodel.unmerge_adapter()\n```\n\nThe [`~LoraModel.add_weighted_adapter`] function is useful for merging multiple LoRAs into a new adapter based on a user provided weighting scheme in the `weights` parameter. Below is an end-to-end example."}


created_at: 2024-08-26 20:30:08
topic: persistent://default/default/dce8dee6-eb49-4277-b4b8-b3dbb5a5778f
id: 977237cb-de61-4243-8e62-0f7ef6a88813
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/peft/developer_guides/lora", "title": "LoRA", "tokens": 4998, "retrieve_doc": true, "source": "peft", "_node_content": "{\"id_\": \"977237cb-de61-4243-8e62-0f7ef6a88813\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"b0064fa4dcc01d18535d58e704f09d04ac4a343d0268b28e4e4449bf362153d7\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"92d4f8d5-7cc9-40f8-90c0-26b645da868c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/peft/developer_guides/lora\", \"title\": \"LoRA\", \"tokens\": 4998, \"retrieve_doc\": true, \"source\": \"peft\"}, \"hash\": \"063d773dea1e287d110183b9e6768f3d285c1a90c8236a421cd46d04258af034\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"13a87b1d-4d27-42a5-8afa-05f421326f70\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ff5a69aac8fe6e5ee50731fc653daa8de1f3c7a98d7b697881807ef5f1579821\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11445, \"end_char_idx\": 14287, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "ref_doc_id": "1c57a8ee-656c-4c3c-bef0-2c2fc7823e4e", "chroma:document": "The [`~LoraModel.merge_and_unload`] function doesn't keep the adapter weights in memory.\n\nBelow is a diagram that explains the intuition of LoRA adapter merging:\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/lora_diagram.png\"/>\n</div>\n\nWe show in the snippets below how to run that using PEFT.\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import PeftModel\n\nbase_model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\npeft_model_id = \"alignment-handbook/zephyr-7b-sft-lora\"\nmodel = PeftModel.from_pretrained(base_model, peft_model_id)\nmodel.merge_and_unload()\n```\n\nIf you need to keep a copy of the weights so you can unmerge the adapter later or delete and load different ones, you should use the [`~LoraModel.merge_adapter`] function instead. Now you have the option to use [`~LoraModel.unmerge_adapter`] to return the base model.\n\n```py\nfrom transformers import AutoModelForCausalLM\nfrom peft import PeftModel\n\nbase_model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\npeft_model_id = \"alignment-handbook/zephyr-7b-sft-lora\"\nmodel = PeftModel.from_pretrained(base_model, peft_model_id)\nmodel.merge_adapter()\n\n# unmerge the LoRA layers from the base model\nmodel.unmerge_adapter()\n```\n\nThe [`~LoraModel.add_weighted_adapter`] function is useful for merging multiple LoRAs into a new adapter based on a user provided weighting scheme in the `weights` parameter. Below is an end-to-end example.\n\nFirst load the base model:\n\n```python\nfrom transformers import AutoModelForCausalLM\nfrom peft import PeftModel\nimport torch\n\nbase_model = AutoModelForCausalLM.from_pretrained(\n    \"mistralai/Mistral-7B-v0.1\", torch_dtype=torch.float16, device_map=\"auto\"\n)\n```\n\nThen we load the first adapter: \n\n```python\npeft_model_id = \"alignment-handbook/zephyr-7b-sft-lora\"\nmodel = PeftModel.from_pretrained(base_model, peft_model_id, adapter_name=\"sft\")\n```\n\nThen load a different adapter and merge it with the first one:\n\n```python\nweighted_adapter_name = \"sft-dpo\"\nmodel.load_adapter(\"alignment-handbook/zephyr-7b-dpo-lora\", adapter_name=\"dpo\")\nmodel.add_weighted_adapter(\n    adapters=[\"sft\", \"dpo\"],\n    weights=[0.7, 0.3],\n    adapter_name=weighted_adapter_name,\n    combination_type=\"linear\"\n)\nmodel.set_adapter(weighted_adapter_name)\n```\n\n<Tip>\n\nThere are several supported methods for `combination_type`. Refer to the [documentation](../package_reference/lora#peft.LoraModel.add_weighted_adapter) for more details. Note that \"svd\" as the `combination_type` is not supported when using `torch.float16` or `torch.bfloat16` as the datatype.\n\n</Tip>\n\nNow, perform inference:\n\n```python\ntokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\n\nprompt = \"Hey, are you conscious? Can you talk to me?\""}


