# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 6741a682-7447-4fb1-8ad9-1177267959a0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"6741a682-7447-4fb1-8ad9-1177267959a0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"278b5932-2ca4-4f7f-97ab-6fe2ba2cda07\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7b6e926217884aa4d45d293ae4066739adfb48fddce666f2b7ac2b49e5479503\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2903, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "# Preprocess\n\n[[open-in-colab]]\n\nBefore you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \ud83e\udd17 Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you'll learn that for:\n\n* Text, use a [Tokenizer](./main_classes/tokenizer) to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors.\n* Speech and audio, use a [Feature extractor](./main_classes/feature_extractor) to extract sequential features from audio waveforms and convert them into tensors.\n* Image inputs use a [ImageProcessor](./main_classes/image_processor) to convert images into tensors.\n* Multimodal inputs, use a [Processor](./main_classes/processors) to combine a tokenizer and a feature extractor or image processor.\n\n<Tip>\n\n`AutoProcessor` **always** works and automatically chooses the correct class for the model you're using, whether you're using a tokenizer, image processor, feature extractor or processor.\n\n</Tip>\n\nBefore you begin, install \ud83e\udd17 Datasets so you can load some datasets to experiment with:\n\n```bash\npip install datasets\n```\n\n## Natural Language Processing\n\n<Youtube id=\"Yffk5aydLzg\"/>\n\nThe main tool for preprocessing textual data is a [tokenizer](main_classes/tokenizer). A tokenizer splits text into *tokens* according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer.\n\n<Tip>\n\nIf you plan on using a pretrained model, it's important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the *vocab*) during pretraining.\n\n</Tip>\n\nGet started by loading a pretrained tokenizer with the [`AutoTokenizer.from_pretrained`] method. This downloads the *vocab* a model was pretrained with:\n\n```py\n>>> from transformers import AutoTokenizer\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-cased\")\n```\n\nThen pass your text to the tokenizer:\n\n```py\n>>> encoded_input = tokenizer(\"Do not meddle in the affairs of wizards, for they are subtle and quick to anger.\")\n>>> print(encoded_input)\n{'input_ids': [101, 2079, 2025, 19960, 10362, 1999, 1996, 3821, 1997, 16657, 1010, 2005, 2027, 2024, 11259, 1998, 4248, 2000, 4963, 1012, 102],\n 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n```\n\nThe tokenizer returns a dictionary with three important items:\n\n* [input_ids](glossary#input-ids) are the indices corresponding to each token in the sentence."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 278b5932-2ca4-4f7f-97ab-6fe2ba2cda07
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"278b5932-2ca4-4f7f-97ab-6fe2ba2cda07\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6741a682-7447-4fb1-8ad9-1177267959a0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cb33f20b07ca1a9f80777824548579572d967acce85051e22799c63e763bccb9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9d2e8ad1-c810-44d1-a31f-4a61d4f474d1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8336be552eec8e5c8f393668167fc2d4ec77acc072931d5e0e258f60c419c4c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1773, \"end_char_idx\": 3814, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the *vocab*) during pretraining.\n\n</Tip>\n\nGet started by loading a pretrained tokenizer with the [`AutoTokenizer.from_pretrained`] method. This downloads the *vocab* a model was pretrained with:\n\n```py\n>>> from transformers import AutoTokenizer\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-cased\")\n```\n\nThen pass your text to the tokenizer:\n\n```py\n>>> encoded_input = tokenizer(\"Do not meddle in the affairs of wizards, for they are subtle and quick to anger.\")\n>>> print(encoded_input)\n{'input_ids': [101, 2079, 2025, 19960, 10362, 1999, 1996, 3821, 1997, 16657, 1010, 2005, 2027, 2024, 11259, 1998, 4248, 2000, 4963, 1012, 102],\n 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n```\n\nThe tokenizer returns a dictionary with three important items:\n\n* [input_ids](glossary#input-ids) are the indices corresponding to each token in the sentence.\n* [attention_mask](glossary#attention-mask) indicates whether a token should be attended to or not.\n* [token_type_ids](glossary#token-type-ids) identifies which sequence a token belongs to when there is more than one sequence.\n\nReturn your input by decoding the `input_ids`:\n\n```py\n>>> tokenizer.decode(encoded_input[\"input_ids\"])\n'[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]'\n```\n\nAs you can see, the tokenizer added two special tokens - `CLS` and `SEP` (classifier and separator) - to the sentence. Not all models need\nspecial tokens, but if they do, the tokenizer automatically adds them for you.\n\nIf there are several sentences you want to preprocess, pass them as a list to the tokenizer:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 9d2e8ad1-c810-44d1-a31f-4a61d4f474d1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9d2e8ad1-c810-44d1-a31f-4a61d4f474d1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"278b5932-2ca4-4f7f-97ab-6fe2ba2cda07\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"82dae28fc3d6d06948bcc12f0153a5e5ba9366d4de83ca10c306720f9c9ff264\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"bc261b6d-bacb-4157-83c5-9b6a4352c005\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bd9f6313951e6714e51bc15e1f90fac0aac4da7a8e937e5b66bb62961712c8b5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2904, \"end_char_idx\": 4970, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "* [attention_mask](glossary#attention-mask) indicates whether a token should be attended to or not.\n* [token_type_ids](glossary#token-type-ids) identifies which sequence a token belongs to when there is more than one sequence.\n\nReturn your input by decoding the `input_ids`:\n\n```py\n>>> tokenizer.decode(encoded_input[\"input_ids\"])\n'[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]'\n```\n\nAs you can see, the tokenizer added two special tokens - `CLS` and `SEP` (classifier and separator) - to the sentence. Not all models need\nspecial tokens, but if they do, the tokenizer automatically adds them for you.\n\nIf there are several sentences you want to preprocess, pass them as a list to the tokenizer:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_inputs = tokenizer(batch_sentences)\n>>> print(encoded_inputs)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1]]}\n```\n\n### Pad\n\nSentences aren't always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special *padding token* to shorter sentences.\n\nSet the `padding` parameter to `True` to pad the shorter sequences in the batch to match the longest sequence:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: bc261b6d-bacb-4157-83c5-9b6a4352c005
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"bc261b6d-bacb-4157-83c5-9b6a4352c005\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9d2e8ad1-c810-44d1-a31f-4a61d4f474d1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"7724906a002dc54e0aeb7f8ce7e2b11bb1633192a02d1b3893a62a537ccd5151\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"86f9fbca-c46b-4d9f-bef5-93c9aadad1f4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3250eb6b2bb63bb28fbd62d8853a0e87b7f5aa26e1ebcc850c8a2524edb5198e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4574, \"end_char_idx\": 6283, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "Padding is a strategy for ensuring tensors are rectangular by adding a special *padding token* to shorter sentences.\n\nSet the `padding` parameter to `True` to pad the shorter sequences in the batch to match the longest sequence:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True)\n>>> print(encoded_input)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]]}\n```\n\nThe first and third sentences are now padded with `0`'s because they are shorter.\n\n### Truncation\n\nOn the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you'll need to truncate the sequence to a shorter length.\n\nSet the `truncation` parameter to `True` to truncate a sequence to the maximum length accepted by the model:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 86f9fbca-c46b-4d9f-bef5-93c9aadad1f4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"86f9fbca-c46b-4d9f-bef5-93c9aadad1f4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"bc261b6d-bacb-4157-83c5-9b6a4352c005\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1505b58ef2d673e96cfe0eecea44a21fc3469d6562e8197937d906ba622d23d3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d0b7dbda-7221-4837-a19e-9d083c4cea81\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f34879afe7db6f923540c8fe90820cb6a8b79ffca3281adadf5751f16a091d58\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5824, \"end_char_idx\": 7603, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "### Truncation\n\nOn the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you'll need to truncate the sequence to a shorter length.\n\nSet the `truncation` parameter to `True` to truncate a sequence to the maximum length accepted by the model:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)\n>>> print(encoded_input)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]]}\n```\n\n<Tip>\n\nCheck out the [Padding and truncation](./pad_truncation) concept guide to learn more different padding and truncation arguments.\n\n</Tip>\n\n### Build tensors\n\nFinally, you want the tokenizer to return the actual tensors that get fed to the model.\n\nSet the `return_tensors` parameter to either `pt` for PyTorch, or `tf` for TensorFlow:\n\n<frameworkcontent>\n<pt>\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: d0b7dbda-7221-4837-a19e-9d083c4cea81
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d0b7dbda-7221-4837-a19e-9d083c4cea81\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"86f9fbca-c46b-4d9f-bef5-93c9aadad1f4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"bee24609fcad865fd1e0cd04c39f87a6448e9eb57ee230b0f18342df56df06c8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"bf94f656-52b0-42d7-9f5c-ea530911f968\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e8b58e4f6ef3714c1b8d88ab0ceb04951452251409f5513703a567bbcb65143b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7208, \"end_char_idx\": 8654, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "</Tip>\n\n### Build tensors\n\nFinally, you want the tokenizer to return the actual tensors that get fed to the model.\n\nSet the `return_tensors` parameter to either `pt` for PyTorch, or `tf` for TensorFlow:\n\n<frameworkcontent>\n<pt>\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors=\"pt\")\n>>> print(encoded_input)\n{'input_ids': tensor([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n                      [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n                      [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]]),\n 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                           [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                           [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]),\n 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                           [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                           [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}\n```\n</pt>\n<tf>\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: bf94f656-52b0-42d7-9f5c-ea530911f968
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"bf94f656-52b0-42d7-9f5c-ea530911f968\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d0b7dbda-7221-4837-a19e-9d083c4cea81\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"475eb98a98a46cefa9f739543358037cf734ac45598a1fa9e5a79ec7b6366342\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"616316e4-e409-4961-815c-e090ccf29c6a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8318013a55868ead22fceda3515200ae8857fa4a7a892b687c376aa66ed8975a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8559, \"end_char_idx\": 10249, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors=\"tf\")\n>>> print(encoded_input)\n{'input_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n       [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n       [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n      dtype=int32)>,\n 'token_type_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>,\n 'attention_mask': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n       [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>}\n```\n</tf>\n</frameworkcontent>\n\n<Tip>\nDifferent pipelines support tokenizer arguments in their `__call__()` differently. `text-2-text-generation` pipelines support (i.e. pass on)\nonly `truncation`. `text-generation` pipelines support `max_length`, `truncation`, `padding` and `add_special_tokens`. \nIn `fill-mask` pipelines, tokenizer arguments can be passed in the `tokenizer_kwargs` argument (dictionary).\n</Tip>\n\n## Audio\n\nFor audio tasks, you'll need a [feature extractor](main_classes/feature_extractor) to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 616316e4-e409-4961-815c-e090ccf29c6a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"616316e4-e409-4961-815c-e090ccf29c6a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"bf94f656-52b0-42d7-9f5c-ea530911f968\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f50103c82d1802a67e0d4d5b02df3542e0a16aa10831d0e848bbd6c073431316\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"be977fd8-b4a2-49ac-b238-028d94c7ea1f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4b45e3f324520fefa6e70718e76fad19d056612febb742f115eaa857818f8e68\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9717, \"end_char_idx\": 12006, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "`text-2-text-generation` pipelines support (i.e. pass on)\nonly `truncation`. `text-generation` pipelines support `max_length`, `truncation`, `padding` and `add_special_tokens`. \nIn `fill-mask` pipelines, tokenizer arguments can be passed in the `tokenizer_kwargs` argument (dictionary).\n</Tip>\n\n## Audio\n\nFor audio tasks, you'll need a [feature extractor](main_classes/feature_extractor) to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors.\n\nLoad the [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:\n\n```py\n>>> from datasets import load_dataset, Audio\n\n>>> dataset = load_dataset(\"PolyAI/minds14\", name=\"en-US\", split=\"train\")\n```\n\nAccess the first element of the `audio` column to take a look at the input. Calling the `audio` column automatically loads and resamples the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 0.        ,  0.00024414, -0.00024414, ..., -0.00024414,\n         0.        ,  0.        ], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 8000}\n```\n\nThis returns three items:\n\n* `array` is the speech signal loaded - and potentially resampled - as a 1D array.\n* `path` points to the location of the audio file.\n* `sampling_rate` refers to how many data points in the speech signal are measured per second.\n\nFor this tutorial, you'll use the [Wav2Vec2](https://huggingface.co/facebook/wav2vec2-base) model. Take a look at the model card, and you'll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data's sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: be977fd8-b4a2-49ac-b238-028d94c7ea1f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"be977fd8-b4a2-49ac-b238-028d94c7ea1f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"616316e4-e409-4961-815c-e090ccf29c6a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eae12348a62d278c14a344b2dae0214b11cd38c6ee262077711804e2e917ebff\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d567f0c7-6f4b-4f95-b6ef-ac766d75037e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0ede54a9d0ee9936ac77410e7cd1850768f9353bcf60dcc8f7575615d2e6e736\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10720, \"end_char_idx\": 13097, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "Calling the `audio` column automatically loads and resamples the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 0.        ,  0.00024414, -0.00024414, ..., -0.00024414,\n         0.        ,  0.        ], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 8000}\n```\n\nThis returns three items:\n\n* `array` is the speech signal loaded - and potentially resampled - as a 1D array.\n* `path` points to the location of the audio file.\n* `sampling_rate` refers to how many data points in the speech signal are measured per second.\n\nFor this tutorial, you'll use the [Wav2Vec2](https://huggingface.co/facebook/wav2vec2-base) model. Take a look at the model card, and you'll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data's sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2. Call the `audio` column again to resample the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 2.3443763e-05,  2.1729663e-04,  2.2145823e-04, ...,\n         3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 16000}\n```\n\nNext, load a feature extractor to normalize and pad the input. When padding textual data, a `0` is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a `0` - interpreted as silence - to `array`.\n\nLoad the feature extractor with [`AutoFeatureExtractor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoFeatureExtractor\n\n>>> feature_extractor = AutoFeatureExtractor.from_pretrained(\"facebook/wav2vec2-base\")\n```\n\nPass the audio `array` to the feature extractor. We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: d567f0c7-6f4b-4f95-b6ef-ac766d75037e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d567f0c7-6f4b-4f95-b6ef-ac766d75037e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"be977fd8-b4a2-49ac-b238-028d94c7ea1f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a38cbad9766cb401559c034fef6a25d91aed93818efca515ee56c30d7e1de8bf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"851c446c-847e-4add-b90c-5d89e0e0a4bf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ce4b82e7e4557e91950b591b48945e9f46d177f24a96da01936e1f6dbe87dfb7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11737, \"end_char_idx\": 14442, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2. Call the `audio` column again to resample the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 2.3443763e-05,  2.1729663e-04,  2.2145823e-04, ...,\n         3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 16000}\n```\n\nNext, load a feature extractor to normalize and pad the input. When padding textual data, a `0` is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a `0` - interpreted as silence - to `array`.\n\nLoad the feature extractor with [`AutoFeatureExtractor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoFeatureExtractor\n\n>>> feature_extractor = AutoFeatureExtractor.from_pretrained(\"facebook/wav2vec2-base\")\n```\n\nPass the audio `array` to the feature extractor. We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur.\n\n```py\n>>> audio_input = [dataset[0][\"audio\"][\"array\"]]\n>>> feature_extractor(audio_input, sampling_rate=16000)\n{'input_values': [array([ 3.8106556e-04,  2.7506407e-03,  2.8015103e-03, ...,\n        5.6335266e-04,  4.6588284e-06, -1.7142107e-04], dtype=float32)]}\n```\n\nJust like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:\n\n```py\n>>> dataset[0][\"audio\"][\"array\"].shape\n(173398,)\n\n>>> dataset[1][\"audio\"][\"array\"].shape\n(106496,)\n```\n\nCreate a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:\n\n```py\n>>> def preprocess_function(examples):\n...     audio_arrays = [x[\"array\"] for x in examples[\"audio\"]]\n...     inputs = feature_extractor(\n...         audio_arrays,\n...         sampling_rate=16000,\n...         padding=True,\n...         max_length=100000,\n...         truncation=True,\n...     )\n...     return inputs\n```\n\nApply the `preprocess_function` to the first few examples in the dataset:\n\n```py\n>>> processed_dataset = preprocess_function(dataset[:5])\n```\n\nThe sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 851c446c-847e-4add-b90c-5d89e0e0a4bf
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"851c446c-847e-4add-b90c-5d89e0e0a4bf\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d567f0c7-6f4b-4f95-b6ef-ac766d75037e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d9cb780779a56717491def3d88e478e65eb7ff6895cd3d9c7b68beb69f32f05b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4188fa5b-0119-43dd-9c60-01dda2f9212d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8babeb15154b929a74c648ac374258ccbd38ffb97d59a18506a171204f37c0a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12960, \"end_char_idx\": 16191, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur.\n\n```py\n>>> audio_input = [dataset[0][\"audio\"][\"array\"]]\n>>> feature_extractor(audio_input, sampling_rate=16000)\n{'input_values': [array([ 3.8106556e-04,  2.7506407e-03,  2.8015103e-03, ...,\n        5.6335266e-04,  4.6588284e-06, -1.7142107e-04], dtype=float32)]}\n```\n\nJust like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:\n\n```py\n>>> dataset[0][\"audio\"][\"array\"].shape\n(173398,)\n\n>>> dataset[1][\"audio\"][\"array\"].shape\n(106496,)\n```\n\nCreate a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:\n\n```py\n>>> def preprocess_function(examples):\n...     audio_arrays = [x[\"array\"] for x in examples[\"audio\"]]\n...     inputs = feature_extractor(\n...         audio_arrays,\n...         sampling_rate=16000,\n...         padding=True,\n...         max_length=100000,\n...         truncation=True,\n...     )\n...     return inputs\n```\n\nApply the `preprocess_function` to the first few examples in the dataset:\n\n```py\n>>> processed_dataset = preprocess_function(dataset[:5])\n```\n\nThe sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!\n\n```py\n>>> processed_dataset[\"input_values\"][0].shape\n(100000,)\n\n>>> processed_dataset[\"input_values\"][1].shape\n(100000,)\n```\n\n## Computer vision\n\nFor computer vision tasks, you'll need an [image processor](main_classes/image_processor) to prepare your dataset for the model.\nImage preprocessing consists of several steps that convert images into the input expected by the model. These steps\ninclude but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.\n\n<Tip>\n\nImage preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation\ntransform image data, but they serve different purposes:\n\n* Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations.\n* Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained.\n\nYou can use any library you like for image augmentation. For image preprocessing, use the `ImageProcessor` associated with the model.\n\n</Tip>\n\nLoad the [food101](https://huggingface.co/datasets/food101) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:\n\n<Tip>\n\nUse \ud83e\udd17 Datasets `split` parameter to only load a small sample from the training split since the dataset is quite large!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 4188fa5b-0119-43dd-9c60-01dda2f9212d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4188fa5b-0119-43dd-9c60-01dda2f9212d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"851c446c-847e-4add-b90c-5d89e0e0a4bf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"77c133edd86254b4e6604a73f6651b639bf1bec5beae8a8d44ae1df6aedd9800\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fc1645b5-4b74-4ac0-bdbf-4116b85ab634\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1410408978c9a36806a714f31d670fdc4c6466b75f566cf9734810c5d6fd0b74\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14389, \"end_char_idx\": 17918, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "You can pass your processed dataset to the model now!\n\n```py\n>>> processed_dataset[\"input_values\"][0].shape\n(100000,)\n\n>>> processed_dataset[\"input_values\"][1].shape\n(100000,)\n```\n\n## Computer vision\n\nFor computer vision tasks, you'll need an [image processor](main_classes/image_processor) to prepare your dataset for the model.\nImage preprocessing consists of several steps that convert images into the input expected by the model. These steps\ninclude but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.\n\n<Tip>\n\nImage preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation\ntransform image data, but they serve different purposes:\n\n* Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations.\n* Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained.\n\nYou can use any library you like for image augmentation. For image preprocessing, use the `ImageProcessor` associated with the model.\n\n</Tip>\n\nLoad the [food101](https://huggingface.co/datasets/food101) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:\n\n<Tip>\n\nUse \ud83e\udd17 Datasets `split` parameter to only load a small sample from the training split since the dataset is quite large!\n\n</Tip>\n\n```py\n>>> from datasets import load_dataset\n\n>>> dataset = load_dataset(\"food101\", split=\"train[:100]\")\n```\n\nNext, take a look at the image with \ud83e\udd17 Datasets [`Image`](https://huggingface.co/docs/datasets/package_reference/main_classes?highlight=image#datasets.Image) feature:\n\n```py\n>>> dataset[0][\"image\"]\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png\"/>\n</div>\n\nLoad the image processor with [`AutoImageProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoImageProcessor\n\n>>> image_processor = AutoImageProcessor.from_pretrained(\"google/vit-base-patch16-224\")\n```\n\nFirst, let's add some image augmentation. You can use any library you prefer, but in this tutorial, we'll use torchvision's [`transforms`](https://pytorch.org/vision/stable/transforms.html) module. If you're interested in using another data augmentation library, learn how in the [Albumentations](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) or [Kornia notebooks](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb).\n\n1. Here we use [`Compose`](https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html) to chain together a couple of\ntransforms - [`RandomResizedCrop`](https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html) and [`ColorJitter`](https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html).\nNote that for resizing, we can get the image size requirements from the `image_processor`."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: fc1645b5-4b74-4ac0-bdbf-4116b85ab634
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"fc1645b5-4b74-4ac0-bdbf-4116b85ab634\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4188fa5b-0119-43dd-9c60-01dda2f9212d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"3e67c67e9fbdc5404f93b4c31ac5eef19f2306d0de16e8650c0900d73ee6ed1b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fd306c5f-e8ff-4320-85f4-4514e4650fe2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4ba95b02d45eb167035a2bb2d6ac07c2f79468a67297cc350a224c5b918ecd96\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16193, \"end_char_idx\": 19543, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "</Tip>\n\n```py\n>>> from datasets import load_dataset\n\n>>> dataset = load_dataset(\"food101\", split=\"train[:100]\")\n```\n\nNext, take a look at the image with \ud83e\udd17 Datasets [`Image`](https://huggingface.co/docs/datasets/package_reference/main_classes?highlight=image#datasets.Image) feature:\n\n```py\n>>> dataset[0][\"image\"]\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png\"/>\n</div>\n\nLoad the image processor with [`AutoImageProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoImageProcessor\n\n>>> image_processor = AutoImageProcessor.from_pretrained(\"google/vit-base-patch16-224\")\n```\n\nFirst, let's add some image augmentation. You can use any library you prefer, but in this tutorial, we'll use torchvision's [`transforms`](https://pytorch.org/vision/stable/transforms.html) module. If you're interested in using another data augmentation library, learn how in the [Albumentations](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) or [Kornia notebooks](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb).\n\n1. Here we use [`Compose`](https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html) to chain together a couple of\ntransforms - [`RandomResizedCrop`](https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html) and [`ColorJitter`](https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html).\nNote that for resizing, we can get the image size requirements from the `image_processor`. For some models, an exact height and\nwidth are expected, for others only the `shortest_edge` is defined.\n\n```py\n>>> from torchvision.transforms import RandomResizedCrop, ColorJitter, Compose\n\n>>> size = (\n...     image_processor.size[\"shortest_edge\"]\n...     if \"shortest_edge\" in image_processor.size\n...     else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n... )\n\n>>> _transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5)])\n```\n\n2. The model accepts [`pixel_values`](model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel.forward.pixel_values)\nas its input. `ImageProcessor` can take care of normalizing the images, and generating appropriate tensors.\nCreate a function that combines image augmentation and image preprocessing for a batch of images and generates `pixel_values`:\n\n```py\n>>> def transforms(examples):\n...     images = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n...     examples[\"pixel_values\"] = image_processor(images, do_resize=False, return_tensors=\"pt\")[\"pixel_values\"]\n...     return examples\n```\n\n<Tip>\n\nIn the example above we set `do_resize=False` because we have already resized the images in the image augmentation transformation,\nand leveraged the `size` attribute from the appropriate `image_processor`. If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: fd306c5f-e8ff-4320-85f4-4514e4650fe2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"fd306c5f-e8ff-4320-85f4-4514e4650fe2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fc1645b5-4b74-4ac0-bdbf-4116b85ab634\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"bbc6e5cc6f03d73e736e09800cd410f2ce36b9467f71570aeda11642bb0a7138\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f7e18b06-79bc-48e3-a80e-4c9825669bcd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dd0abd2823e1c334ee40889278d1ca109fe21917312dd7f23cf17b401dd8e964\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17828, \"end_char_idx\": 20921, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "Note that for resizing, we can get the image size requirements from the `image_processor`. For some models, an exact height and\nwidth are expected, for others only the `shortest_edge` is defined.\n\n```py\n>>> from torchvision.transforms import RandomResizedCrop, ColorJitter, Compose\n\n>>> size = (\n...     image_processor.size[\"shortest_edge\"]\n...     if \"shortest_edge\" in image_processor.size\n...     else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n... )\n\n>>> _transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5)])\n```\n\n2. The model accepts [`pixel_values`](model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel.forward.pixel_values)\nas its input. `ImageProcessor` can take care of normalizing the images, and generating appropriate tensors.\nCreate a function that combines image augmentation and image preprocessing for a batch of images and generates `pixel_values`:\n\n```py\n>>> def transforms(examples):\n...     images = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n...     examples[\"pixel_values\"] = image_processor(images, do_resize=False, return_tensors=\"pt\")[\"pixel_values\"]\n...     return examples\n```\n\n<Tip>\n\nIn the example above we set `do_resize=False` because we have already resized the images in the image augmentation transformation,\nand leveraged the `size` attribute from the appropriate `image_processor`. If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3. Then use \ud83e\udd17 Datasets[`~datasets.Dataset.set_transform`] to apply the transforms on the fly:\n```py\n>>> dataset.set_transform(transforms)\n```\n\n4. Now when you access the image, you'll notice the image processor has added `pixel_values`. You can pass your processed dataset to the model now!\n\n```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: f7e18b06-79bc-48e3-a80e-4c9825669bcd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f7e18b06-79bc-48e3-a80e-4c9825669bcd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fd306c5f-e8ff-4320-85f4-4514e4650fe2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"77dde90ed4160e8c6d60b85c74af9a99898accf63c156343bb79f4fc92bc0584\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fc6a0b42-df99-446d-b8e8-85f4eb79cab6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c39f41d58c94f8d1a15fe5885021a22add22dfc84ed9e62378aff7d4f79c0e70\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19240, \"end_char_idx\": 21570, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3. Then use \ud83e\udd17 Datasets[`~datasets.Dataset.set_transform`] to apply the transforms on the fly:\n```py\n>>> dataset.set_transform(transforms)\n```\n\n4. Now when you access the image, you'll notice the image processor has added `pixel_values`. You can pass your processed dataset to the model now!\n\n```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together.\n\n```py\n>>> def collate_fn(batch):\n...     pixel_values = [item[\"pixel_values\"] for item in batch]\n...     encoding = image_processor.pad(pixel_values, return_tensors=\"pt\")\n...     labels = [item[\"labels\"] for item in batch]\n...     batch = {}\n...     batch[\"pixel_values\"] = encoding[\"pixel_values\"]\n...     batch[\"pixel_mask\"] = encoding[\"pixel_mask\"]\n...     batch[\"labels\"] = labels\n...     return batch\n```\n\n## Multimodal\n\nFor tasks involving multimodal inputs, you'll need a [processor](main_classes/processors) to prepare your dataset for the model. A processor couples together two processing objects such as tokenizer and feature extractor."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: fc6a0b42-df99-446d-b8e8-85f4eb79cab6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"fc6a0b42-df99-446d-b8e8-85f4eb79cab6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f7e18b06-79bc-48e3-a80e-4c9825669bcd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0063149ed445dc0c76f115253cd977d064d5fc2df24b2160ce89d9c107628802\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b3afd71d-5a31-4f71-80a1-372f14adacdd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7196abb711ab229bf55d28e6fc41efa5416af045ca48ec302caa817d181487d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19833, \"end_char_idx\": 22892, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together.\n\n```py\n>>> def collate_fn(batch):\n...     pixel_values = [item[\"pixel_values\"] for item in batch]\n...     encoding = image_processor.pad(pixel_values, return_tensors=\"pt\")\n...     labels = [item[\"labels\"] for item in batch]\n...     batch = {}\n...     batch[\"pixel_values\"] = encoding[\"pixel_values\"]\n...     batch[\"pixel_mask\"] = encoding[\"pixel_mask\"]\n...     batch[\"labels\"] = labels\n...     return batch\n```\n\n## Multimodal\n\nFor tasks involving multimodal inputs, you'll need a [processor](main_classes/processors) to prepare your dataset for the model. A processor couples together two processing objects such as tokenizer and feature extractor.\n\nLoad the [LJ Speech](https://huggingface.co/datasets/lj_speech) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):\n\n```py\n>>> from datasets import load_dataset\n\n>>> lj_speech = load_dataset(\"lj_speech\", split=\"train\")\n```\n\nFor ASR, you're mainly focused on `audio` and `text` so you can remove the other columns:\n\n```py\n>>> lj_speech = lj_speech.map(remove_columns=[\"file\", \"id\", \"normalized_text\"])\n```\n\nNow take a look at the `audio` and `text` columns:\n\n```py\n>>> lj_speech[0][\"audio\"]\n{'array': array([-7.3242188e-04, -7.6293945e-04, -6.4086914e-04, ...,\n         7.3242188e-04,  2.1362305e-04,  6.1035156e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav',\n 'sampling_rate': 22050}\n\n>>> lj_speech[0][\"text\"]\n'Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition'\n```\n\nRemember you should always [resample](preprocessing#audio) your audio dataset's sampling rate to match the sampling rate of the dataset used to pretrain a model!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: b3afd71d-5a31-4f71-80a1-372f14adacdd
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"b3afd71d-5a31-4f71-80a1-372f14adacdd\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c77c70d4-491e-4d74-ae17-8f1df6277010\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fc6a0b42-df99-446d-b8e8-85f4eb79cab6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e286f602d198cd0c06b5064766d12afca550545a8737abdf2be565c382f6eb20\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21572, \"end_char_idx\": 23806, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "ref_doc_id": "c77c70d4-491e-4d74-ae17-8f1df6277010", "chroma:document": "Load the [LJ Speech](https://huggingface.co/datasets/lj_speech) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):\n\n```py\n>>> from datasets import load_dataset\n\n>>> lj_speech = load_dataset(\"lj_speech\", split=\"train\")\n```\n\nFor ASR, you're mainly focused on `audio` and `text` so you can remove the other columns:\n\n```py\n>>> lj_speech = lj_speech.map(remove_columns=[\"file\", \"id\", \"normalized_text\"])\n```\n\nNow take a look at the `audio` and `text` columns:\n\n```py\n>>> lj_speech[0][\"audio\"]\n{'array': array([-7.3242188e-04, -7.6293945e-04, -6.4086914e-04, ...,\n         7.3242188e-04,  2.1362305e-04,  6.1035156e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav',\n 'sampling_rate': 22050}\n\n>>> lj_speech[0][\"text\"]\n'Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition'\n```\n\nRemember you should always [resample](preprocessing#audio) your audio dataset's sampling rate to match the sampling rate of the dataset used to pretrain a model!\n\n```py\n>>> lj_speech = lj_speech.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\nLoad a processor with [`AutoProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoProcessor\n\n>>> processor = AutoProcessor.from_pretrained(\"facebook/wav2vec2-base-960h\")\n```\n\n1. Create a function to process the audio data contained in `array` to `input_values`, and tokenize `text` to `labels`. These are the inputs to the model:\n\n```py\n>>> def prepare_dataset(example):\n...     audio = example[\"audio\"]\n\n...     example.update(processor(audio=audio[\"array\"], text=example[\"text\"], sampling_rate=16000))\n\n...     return example\n```\n\n2. Apply the `prepare_dataset` function to a sample:\n\n```py\n>>> prepare_dataset(lj_speech[0])\n```\n\nThe processor has now added `input_values` and `labels`, and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: c3dedbf7-9b19-4664-b540-e5e5406b0e44
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/sagemaker", "title": "Run training on Amazon SageMaker", "tokens": 278, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"c3dedbf7-9b19-4664-b540-e5e5406b0e44\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/sagemaker\", \"title\": \"Run training on Amazon SageMaker\", \"tokens\": 278, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9d7a918b-1ebb-439d-abd1-26f7b183ffa5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/sagemaker\", \"title\": \"Run training on Amazon SageMaker\", \"tokens\": 278, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"7698b2ff621f3e0a64efa530a64dc41f68dc6ea4f228f13ba9b6e5f358029a6b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1220, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9d7a918b-1ebb-439d-abd1-26f7b183ffa5", "doc_id": "9d7a918b-1ebb-439d-abd1-26f7b183ffa5", "ref_doc_id": "9d7a918b-1ebb-439d-abd1-26f7b183ffa5", "chroma:document": "<!---\nCopyright 2020 The HuggingFace Team. All rights reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\n\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n\n-->\n\n# Run training on Amazon SageMaker\n\nThe documentation has been moved to [hf.co/docs/sagemaker](https://huggingface.co/docs/sagemaker). This page will be removed in `transformers` 5.0. \n\n### Table of Content\n\n- [Train Hugging Face models on Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/train)\n- [Deploy Hugging Face models to Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/inference)"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 90ae211d-08ca-4619-90f4-9ef328c55280
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2341, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"90ae211d-08ca-4619-90f4-9ef328c55280\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95e4d21b-5430-4814-a1a7-a0751960cd7f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"60d389b98b70973e18401b03836e98d77a216158a1e16fe011c2142e03543ba8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8790ede3-9801-416a-a28b-d7d67d3a4da0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b6923e4934a3ea39b940ba629f42969d86c162b763559003e2725dbe09ee9fba\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3390, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "ref_doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "chroma:document": "# Share a model\n\nThe last two tutorials showed how you can fine-tune a model with PyTorch, Keras, and \ud83e\udd17 Accelerate for distributed setups. The next step is to share your model with the community! At Hugging Face, we believe in openly sharing knowledge and resources to democratize artificial intelligence for everyone. We encourage you to consider sharing your model with the community to help others save time and resources.\n\nIn this tutorial, you will learn two methods for sharing a trained or fine-tuned model on the [Model Hub](https://huggingface.co/models):\n\n- Programmatically push your files to the Hub.\n- Drag-and-drop your files to the Hub with the web interface.\n\n<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/XvSGPZFEjDY\" title=\"YouTube video player\"\nframeborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope;\npicture-in-picture\" allowfullscreen></iframe>\n\n<Tip>\n\nTo share a model with the community, you need an account on [huggingface.co](https://huggingface.co/join). You can also join an existing organization or create a new one.\n\n</Tip>\n\n## Repository features\n\nEach repository on the Model Hub behaves like a typical GitHub repository. Our repositories offer versioning, commit history, and the ability to visualize differences.\n\nThe Model Hub's built-in versioning is based on git and [git-lfs](https://git-lfs.github.com/). In other words, you can treat one model as one repository, enabling greater access control and scalability. Version control allows *revisions*, a method for pinning a specific version of a model with a commit hash, tag or branch.\n\nAs a result, you can load a specific model version with the `revision` parameter:\n\n```py\n>>> model = AutoModel.from_pretrained(\n...     \"julien-c/EsperBERTo-small\", revision=\"v2.0.1\"  # tag name, or branch name, or commit hash\n... )\n```\n\nFiles are also easily edited in a repository, and you can view the commit history as well as the difference:\n\n![vis_diff](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vis_diff.png)\n\n## Setup\n\nBefore sharing a model to the Hub, you will need your Hugging Face credentials. If you have access to a terminal, run the following command in the virtual environment where \ud83e\udd17 Transformers is installed. This will store your access token in your Hugging Face cache folder (`~/.cache/` by default):\n\n```bash\nhuggingface-cli login\n```\n\nIf you are using a notebook like Jupyter or Colaboratory, make sure you have the [`huggingface_hub`](https://huggingface.co/docs/hub/adding-a-library) library installed. This library allows you to programmatically interact with the Hub.\n\n```bash\npip install huggingface_hub\n```\n\nThen use `notebook_login` to sign-in to the Hub, and follow the link [here](https://huggingface.co/settings/token) to generate a token to login with:\n\n```py\n>>> from huggingface_hub import notebook_login\n\n>>> notebook_login()\n```\n\n## Convert a model for all frameworks\n\nTo ensure your model can be used by someone working with a different framework, we recommend you convert and upload your model with both PyTorch and TensorFlow checkpoints. While users are still able to load your model from a different framework if you skip this step, it will be slower because \ud83e\udd17 Transformers will need to convert the checkpoint on-the-fly.\n\nConverting a checkpoint for another framework is easy."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 8790ede3-9801-416a-a28b-d7d67d3a4da0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2341, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8790ede3-9801-416a-a28b-d7d67d3a4da0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95e4d21b-5430-4814-a1a7-a0751960cd7f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"60d389b98b70973e18401b03836e98d77a216158a1e16fe011c2142e03543ba8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"90ae211d-08ca-4619-90f4-9ef328c55280\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"57b8c29332d2ed8f5a0c549207b81bb957198dc058215c62bad3e579996eb51d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8381d8d6-a7a2-41ca-99ef-b93e17652b9a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9a8fda1c3a5dfa0893a2d863d0e8026ee225667854bf81254c53b2af6b804e33\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1982, \"end_char_idx\": 4908, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "ref_doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "chroma:document": "[vis_diff](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vis_diff.png)\n\n## Setup\n\nBefore sharing a model to the Hub, you will need your Hugging Face credentials. If you have access to a terminal, run the following command in the virtual environment where \ud83e\udd17 Transformers is installed. This will store your access token in your Hugging Face cache folder (`~/.cache/` by default):\n\n```bash\nhuggingface-cli login\n```\n\nIf you are using a notebook like Jupyter or Colaboratory, make sure you have the [`huggingface_hub`](https://huggingface.co/docs/hub/adding-a-library) library installed. This library allows you to programmatically interact with the Hub.\n\n```bash\npip install huggingface_hub\n```\n\nThen use `notebook_login` to sign-in to the Hub, and follow the link [here](https://huggingface.co/settings/token) to generate a token to login with:\n\n```py\n>>> from huggingface_hub import notebook_login\n\n>>> notebook_login()\n```\n\n## Convert a model for all frameworks\n\nTo ensure your model can be used by someone working with a different framework, we recommend you convert and upload your model with both PyTorch and TensorFlow checkpoints. While users are still able to load your model from a different framework if you skip this step, it will be slower because \ud83e\udd17 Transformers will need to convert the checkpoint on-the-fly.\n\nConverting a checkpoint for another framework is easy. Make sure you have PyTorch and TensorFlow installed (see [here](installation) for installation instructions), and then find the specific model for your task in the other framework.\n\n<frameworkcontent>\n<pt>\nSpecify `from_tf=True` to convert a checkpoint from TensorFlow to PyTorch:\n\n```py\n>>> pt_model = DistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_tf=True)\n>>> pt_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</pt>\n<tf>\nSpecify `from_pt=True` to convert a checkpoint from PyTorch to TensorFlow:\n\n```py\n>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_pt=True)\n```\n\nThen you can save your new TensorFlow model with its new checkpoint:\n\n```py\n>>> tf_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</tf>\n<jax>\nIf a model is available in Flax, you can also convert a checkpoint from PyTorch to Flax:\n\n```py\n>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(\n...     \"path/to/awesome-name-you-picked\", from_pt=True\n... )\n```\n</jax>\n</frameworkcontent>\n\n## Push a model during training\n\n<frameworkcontent>\n<pt>\n<Youtube id=\"Z1-XMy-GNLQ\"/>\n\nSharing a model to the Hub is as simple as adding an extra parameter or callback. Remember from the [fine-tuning tutorial](training), the [`TrainingArguments`] class is where you specify hyperparameters and additional training options. One of these training options includes the ability to push a model directly to the Hub."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 8381d8d6-a7a2-41ca-99ef-b93e17652b9a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2341, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8381d8d6-a7a2-41ca-99ef-b93e17652b9a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95e4d21b-5430-4814-a1a7-a0751960cd7f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"60d389b98b70973e18401b03836e98d77a216158a1e16fe011c2142e03543ba8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8790ede3-9801-416a-a28b-d7d67d3a4da0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"4f2a8c6a6035561998831b678f85821e02e3e30f797b1681d263e47da81e4a78\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d1df8d3e-9b87-47cd-ab21-1fe00a91e103\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"359803c6316cafb665e7d0c7efae85dc176287329b68d5a91cf8411c971d0036\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3336, \"end_char_idx\": 6479, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "ref_doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "chroma:document": "Converting a checkpoint for another framework is easy. Make sure you have PyTorch and TensorFlow installed (see [here](installation) for installation instructions), and then find the specific model for your task in the other framework.\n\n<frameworkcontent>\n<pt>\nSpecify `from_tf=True` to convert a checkpoint from TensorFlow to PyTorch:\n\n```py\n>>> pt_model = DistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_tf=True)\n>>> pt_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</pt>\n<tf>\nSpecify `from_pt=True` to convert a checkpoint from PyTorch to TensorFlow:\n\n```py\n>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_pt=True)\n```\n\nThen you can save your new TensorFlow model with its new checkpoint:\n\n```py\n>>> tf_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</tf>\n<jax>\nIf a model is available in Flax, you can also convert a checkpoint from PyTorch to Flax:\n\n```py\n>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(\n...     \"path/to/awesome-name-you-picked\", from_pt=True\n... )\n```\n</jax>\n</frameworkcontent>\n\n## Push a model during training\n\n<frameworkcontent>\n<pt>\n<Youtube id=\"Z1-XMy-GNLQ\"/>\n\nSharing a model to the Hub is as simple as adding an extra parameter or callback. Remember from the [fine-tuning tutorial](training), the [`TrainingArguments`] class is where you specify hyperparameters and additional training options. One of these training options includes the ability to push a model directly to the Hub. Set `push_to_hub=True` in your [`TrainingArguments`]:\n\n```py\n>>> training_args = TrainingArguments(output_dir=\"my-awesome-model\", push_to_hub=True)\n```\n\nPass your training arguments as usual to [`Trainer`]:\n\n```py\n>>> trainer = Trainer(\n...     model=model,\n...     args=training_args,\n...     train_dataset=small_train_dataset,\n...     eval_dataset=small_eval_dataset,\n...     compute_metrics=compute_metrics,\n... )\n```\n\nAfter you fine-tune your model, call [`~transformers.Trainer.push_to_hub`] on [`Trainer`] to push the trained model to the Hub. \ud83e\udd17 Transformers will even automatically add training hyperparameters, training results and framework versions to your model card!\n\n```py\n>>> trainer.push_to_hub()\n```\n</pt>\n<tf>\nShare a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\n\n- An output directory for your model.\n- A tokenizer.\n- The `hub_model_id`, which is your Hub username and model name.\n\n```py\n>>> from transformers import PushToHubCallback\n\n>>> push_to_hub_callback = PushToHubCallback(\n...     output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\n... )\n```\n\nAdd the callback to [`fit`](https://keras.io/api/models/model_training_apis/), and \ud83e\udd17 Transformers will push the trained model to the Hub:\n\n```py\n>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)\n```\n</tf>\n</frameworkcontent>\n\n## Use the `push_to_hub` function\n\nYou can also call `push_to_hub` directly on your model to upload it to the Hub."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: d1df8d3e-9b87-47cd-ab21-1fe00a91e103
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2341, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d1df8d3e-9b87-47cd-ab21-1fe00a91e103\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95e4d21b-5430-4814-a1a7-a0751960cd7f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"60d389b98b70973e18401b03836e98d77a216158a1e16fe011c2142e03543ba8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8381d8d6-a7a2-41ca-99ef-b93e17652b9a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1e1c0e14cb782f47f87f77a50ff9ee27972665a14048909187b1b9f1446f512b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2b462324-57e6-4136-b2f2-a10c3d534fc6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e79843da10f7db55e0b89d3b645885e719cdc4703c2e0c9759e2c7a14ae32657\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4821, \"end_char_idx\": 7965, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "ref_doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "chroma:document": "One of these training options includes the ability to push a model directly to the Hub. Set `push_to_hub=True` in your [`TrainingArguments`]:\n\n```py\n>>> training_args = TrainingArguments(output_dir=\"my-awesome-model\", push_to_hub=True)\n```\n\nPass your training arguments as usual to [`Trainer`]:\n\n```py\n>>> trainer = Trainer(\n...     model=model,\n...     args=training_args,\n...     train_dataset=small_train_dataset,\n...     eval_dataset=small_eval_dataset,\n...     compute_metrics=compute_metrics,\n... )\n```\n\nAfter you fine-tune your model, call [`~transformers.Trainer.push_to_hub`] on [`Trainer`] to push the trained model to the Hub. \ud83e\udd17 Transformers will even automatically add training hyperparameters, training results and framework versions to your model card!\n\n```py\n>>> trainer.push_to_hub()\n```\n</pt>\n<tf>\nShare a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\n\n- An output directory for your model.\n- A tokenizer.\n- The `hub_model_id`, which is your Hub username and model name.\n\n```py\n>>> from transformers import PushToHubCallback\n\n>>> push_to_hub_callback = PushToHubCallback(\n...     output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\n... )\n```\n\nAdd the callback to [`fit`](https://keras.io/api/models/model_training_apis/), and \ud83e\udd17 Transformers will push the trained model to the Hub:\n\n```py\n>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)\n```\n</tf>\n</frameworkcontent>\n\n## Use the `push_to_hub` function\n\nYou can also call `push_to_hub` directly on your model to upload it to the Hub.\n\nSpecify your model name in `push_to_hub`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-model\")\n```\n\nThis creates a repository under your username with the model name `my-awesome-model`. Users can now load your model with the `from_pretrained` function:\n\n```py\n>>> from transformers import AutoModel\n\n>>> model = AutoModel.from_pretrained(\"your_username/my-awesome-model\")\n```\n\nIf you belong to an organization and want to push your model under the organization name instead, just add it to the `repo_id`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-org/my-awesome-model\")\n```\n\nThe `push_to_hub` function can also be used to add other files to a model repository. For example, add a tokenizer to a model repository:\n\n```py\n>>> tokenizer.push_to_hub(\"my-awesome-model\")\n```\n\nOr perhaps you'd like to add the TensorFlow version of your fine-tuned PyTorch model:\n\n```py\n>>> tf_model.push_to_hub(\"my-awesome-model\")\n```\n\nNow when you navigate to your Hugging Face profile, you should see your newly created model repository. Clicking on the **Files** tab will display all the files you've uploaded to the repository.\n\nFor more details on how to create and upload files to a repository, refer to the Hub documentation [here](https://huggingface.co/docs/hub/how-to-upstream).\n\n## Upload with the web interface\n\nUsers who prefer a no-code approach are able to upload a model through the Hub's web interface. Visit [huggingface.co/new](https://huggingface.co/new) to create a new repository:\n\n!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 2b462324-57e6-4136-b2f2-a10c3d534fc6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2341, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2b462324-57e6-4136-b2f2-a10c3d534fc6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"95e4d21b-5430-4814-a1a7-a0751960cd7f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"60d389b98b70973e18401b03836e98d77a216158a1e16fe011c2142e03543ba8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d1df8d3e-9b87-47cd-ab21-1fe00a91e103\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2341, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ed050444d4856a27ac747df271231c87d2eb738171580b84a5bd1d2a99cd3322\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6481, \"end_char_idx\": 9462, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "ref_doc_id": "95e4d21b-5430-4814-a1a7-a0751960cd7f", "chroma:document": "Specify your model name in `push_to_hub`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-model\")\n```\n\nThis creates a repository under your username with the model name `my-awesome-model`. Users can now load your model with the `from_pretrained` function:\n\n```py\n>>> from transformers import AutoModel\n\n>>> model = AutoModel.from_pretrained(\"your_username/my-awesome-model\")\n```\n\nIf you belong to an organization and want to push your model under the organization name instead, just add it to the `repo_id`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-org/my-awesome-model\")\n```\n\nThe `push_to_hub` function can also be used to add other files to a model repository. For example, add a tokenizer to a model repository:\n\n```py\n>>> tokenizer.push_to_hub(\"my-awesome-model\")\n```\n\nOr perhaps you'd like to add the TensorFlow version of your fine-tuned PyTorch model:\n\n```py\n>>> tf_model.push_to_hub(\"my-awesome-model\")\n```\n\nNow when you navigate to your Hugging Face profile, you should see your newly created model repository. Clicking on the **Files** tab will display all the files you've uploaded to the repository.\n\nFor more details on how to create and upload files to a repository, refer to the Hub documentation [here](https://huggingface.co/docs/hub/how-to-upstream).\n\n## Upload with the web interface\n\nUsers who prefer a no-code approach are able to upload a model through the Hub's web interface. Visit [huggingface.co/new](https://huggingface.co/new) to create a new repository:\n\n![new_model_repo](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/new_model_repo.png)\n\nFrom here, add some information about your model:\n\n- Select the **owner** of the repository. This can be yourself or any of the organizations you belong to.\n- Pick a name for your model, which will also be the repository name.\n- Choose whether your model is public or private.\n- Specify the license usage for your model.\n\nNow click on the **Files** tab and click on the **Add file** button to upload a new file to your repository. Then drag-and-drop a file to upload and add a commit message.\n\n![upload_file](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/upload_file.png)\n\n## Add a model card\n\nTo make sure users understand your model's capabilities, limitations, potential biases and ethical considerations, please add a model card to your repository. The model card is defined in the `README.md` file. You can add a model card by:\n\n* Manually creating and uploading a `README.md` file.\n* Clicking on the **Edit model card** button in your model repository.\n\nTake a look at the DistilBert [model card](https://huggingface.co/distilbert/distilbert-base-uncased) for a good example of the type of information a model card should include. For more details about other options you can control in the `README.md` file such as a model's carbon footprint or widget examples, refer to the documentation [here](https://huggingface.co/docs/hub/models-cards)."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 9cbf597b-562c-47df-97c6-33d0bdbd9b00
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9cbf597b-562c-47df-97c6-33d0bdbd9b00\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"807c3bb3-e0b5-40ca-8c6b-f78805efdf85\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9acc167e50e9a635700da48f9c51c2e752c556256a126da90553b5316b2be87b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2914, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "# GPU inference\n\nGPUs are the standard choice of hardware for machine learning, unlike CPUs, because they are optimized for memory bandwidth and parallelism. To keep up with the larger sizes of modern models or to run these large models on existing and older hardware, there are several optimizations you can use to speed up GPU inference. In this guide, you'll learn how to use FlashAttention-2 (a more memory-efficient attention mechanism), BetterTransformer (a PyTorch native fastpath execution), and bitsandbytes to quantize your model to a lower precision. Finally, learn how to use \ud83e\udd17 Optimum to accelerate inference with ONNX Runtime on Nvidia and AMD GPUs.\n\n<Tip>\n\nThe majority of the optimizations described here also apply to multi-GPU setups!\n\n</Tip>\n\n## FlashAttention-2\n\n<Tip>\n\nFlashAttention-2 is experimental and may change considerably in future versions.\n\n</Tip>\n\n[FlashAttention-2](https://huggingface.co/papers/2205.14135) is a faster and more efficient implementation of the standard attention mechanism that can significantly speedup inference by:\n\n1. additionally parallelizing the attention computation over sequence length\n2. partitioning the work between GPU threads to reduce communication and shared memory reads/writes between them\n\nFlashAttention-2 is currently supported for the following architectures:\n* [Bark](https://huggingface.co/docs/transformers/model_doc/bark#transformers.BarkModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DistilBert](https://huggingface.co/docs/transformers/model_doc/distilbert#transformers.DistilBertModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeo](https://huggingface.co/docs/transformers/model_doc/gpt_neo#transformers.GPTNeoModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj#transformers.GPTJModel)\n* [Idefics2](https://huggingface.co/docs/transformers/model_doc/idefics2#transformers.Idefics2Model)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 807c3bb3-e0b5-40ca-8c6b-f78805efdf85
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"807c3bb3-e0b5-40ca-8c6b-f78805efdf85\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9cbf597b-562c-47df-97c6-33d0bdbd9b00\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"16815aefd8397ab4e49fd404de266f531d973053bcfd90dc2a1a9f208b019a60\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c2df8d2a-7028-4ad3-9379-43e0506774a2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5284c40f91c497f5a1f21011dc8fcca97ac1bbf4c1ea7b25d1c270d2273c8b5b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1730, \"end_char_idx\": 4114, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DistilBert](https://huggingface.co/docs/transformers/model_doc/distilbert#transformers.DistilBertModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeo](https://huggingface.co/docs/transformers/model_doc/gpt_neo#transformers.GPTNeoModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj#transformers.GPTJModel)\n* [Idefics2](https://huggingface.co/docs/transformers/model_doc/idefics2#transformers.Idefics2Model)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [Llava](https://huggingface.co/docs/transformers/model_doc/llava)\n* [Llava-NeXT](https://huggingface.co/docs/transformers/model_doc/llava_next)\n* [Llava-NeXT-Video](https://huggingface.co/docs/transformers/model_doc/llava_next_video)\n* [VipLlava](https://huggingface.co/docs/transformers/model_doc/vipllava)\n* [VideoLlava](https://huggingface.co/docs/transformers/model_doc/video_llava)\n* [M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)\n* [MBart](https://huggingface.co/docs/transformers/model_doc/mbart#transformers.MBartModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [Nemotron](https://huggingface."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: c2df8d2a-7028-4ad3-9379-43e0506774a2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"c2df8d2a-7028-4ad3-9379-43e0506774a2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"807c3bb3-e0b5-40ca-8c6b-f78805efdf85\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a2d3ac4ed3e51eeda70a2be1a21fc2e94d0692779cd8e66e6cd66bbbdfcf5dc0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"aa750ee1-c4ff-4197-8fc8-5e0d4552826e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e12c2e5ae5eec2351cb99dc0d6744aecff323f08617325e5ca05977a79d06aab\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2863, \"end_char_idx\": 5284, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [Llava](https://huggingface.co/docs/transformers/model_doc/llava)\n* [Llava-NeXT](https://huggingface.co/docs/transformers/model_doc/llava_next)\n* [Llava-NeXT-Video](https://huggingface.co/docs/transformers/model_doc/llava_next_video)\n* [VipLlava](https://huggingface.co/docs/transformers/model_doc/vipllava)\n* [VideoLlava](https://huggingface.co/docs/transformers/model_doc/video_llava)\n* [M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)\n* [MBart](https://huggingface.co/docs/transformers/model_doc/mbart#transformers.MBartModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)\n* [NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [OPT](https://huggingface.co/docs/transformers/model_doc/opt#transformers.OPTModel)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2Audio](https://huggingface.co/docs/transformers/model_doc/qwen2_audio#transformers.Qwen2AudioEncoder)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel)\n* [Whisper](https://huggingface."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: aa750ee1-c4ff-4197-8fc8-5e0d4552826e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"aa750ee1-c4ff-4197-8fc8-5e0d4552826e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c2df8d2a-7028-4ad3-9379-43e0506774a2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ca0f5eb7741caf93c830e975b2042c83e9506d8beeed13d61135c341b2a84866\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0bff2c01-8569-40c6-adb5-6c84f029457d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"01b7ece2b090935049f47c8c9f39eb667f67a66d8d1b25276b7da1be2ef7e2b0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4060, \"end_char_idx\": 6503, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "MusicgenMelodyModel)\n* [Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)\n* [NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [OPT](https://huggingface.co/docs/transformers/model_doc/opt#transformers.OPTModel)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2Audio](https://huggingface.co/docs/transformers/model_doc/qwen2_audio#transformers.Qwen2AudioEncoder)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n\nYou can request to add FlashAttention-2 support for another model by opening a GitHub Issue or Pull Request.\n\nBefore you begin, make sure you have FlashAttention-2 installed.\n\n<hfoptions id=\"install\">\n<hfoption id=\"NVIDIA\">\n\n```bash\npip install flash-attn --no-build-isolation\n```\n\nWe strongly suggest referring to the detailed [installation instructions](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features) to learn more about supported hardware and data types!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 0bff2c01-8569-40c6-adb5-6c84f029457d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0bff2c01-8569-40c6-adb5-6c84f029457d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"aa750ee1-c4ff-4197-8fc8-5e0d4552826e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb74660e50026ed96eb75427f49774c04bacb5905986ee147adc42a443b346e0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e4443492-bb17-4fd9-9471-ff3515188706\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2b03646c76fb23028f0bcde26f2ad2e40b1ad2163ee043c48c554fa4d3ae26a0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5138, \"end_char_idx\": 7848, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "Qwen2MoeModel)\n* [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n\nYou can request to add FlashAttention-2 support for another model by opening a GitHub Issue or Pull Request.\n\nBefore you begin, make sure you have FlashAttention-2 installed.\n\n<hfoptions id=\"install\">\n<hfoption id=\"NVIDIA\">\n\n```bash\npip install flash-attn --no-build-isolation\n```\n\nWe strongly suggest referring to the detailed [installation instructions](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features) to learn more about supported hardware and data types!\n\n</hfoption>\n<hfoption id=\"AMD\">\n\nFlashAttention-2 is also supported on AMD GPUs and current support is limited to **Instinct MI210**, **Instinct MI250** and **Instinct MI300**. We strongly suggest using this [Dockerfile](https://github.com/huggingface/optimum-amd/tree/main/docker/transformers-pytorch-amd-gpu-flash/Dockerfile) to use FlashAttention-2 on AMD GPUs.\n\n</hfoption>\n</hfoptions>\n\nTo enable FlashAttention-2, pass the argument `attn_implementation=\"flash_attention_2\"` to [`~AutoModelForCausalLM.from_pretrained`]:\n\n```python\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    torch_dtype=torch.bfloat16,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n<Tip>\n\nFlashAttention-2 can only be used when the model's dtype is `fp16` or `bf16`. Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: e4443492-bb17-4fd9-9471-ff3515188706
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"e4443492-bb17-4fd9-9471-ff3515188706\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0bff2c01-8569-40c6-adb5-6c84f029457d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f634b3cf21a6980c3565b1e0d6a7f11aaf2a20ff691635eff906e774aaa84f95\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c38792b5-2aed-4e5c-841b-4ebff5c0222a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3329164b29608d66e7683702b3159761ea523b1a556afb98036610bf6198e2b4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6505, \"end_char_idx\": 9132, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "</hfoption>\n<hfoption id=\"AMD\">\n\nFlashAttention-2 is also supported on AMD GPUs and current support is limited to **Instinct MI210**, **Instinct MI250** and **Instinct MI300**. We strongly suggest using this [Dockerfile](https://github.com/huggingface/optimum-amd/tree/main/docker/transformers-pytorch-amd-gpu-flash/Dockerfile) to use FlashAttention-2 on AMD GPUs.\n\n</hfoption>\n</hfoptions>\n\nTo enable FlashAttention-2, pass the argument `attn_implementation=\"flash_attention_2\"` to [`~AutoModelForCausalLM.from_pretrained`]:\n\n```python\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    torch_dtype=torch.bfloat16,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n<Tip>\n\nFlashAttention-2 can only be used when the model's dtype is `fp16` or `bf16`. Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference. For example, you can combine FlashAttention-2 with 8-bit or 4-bit quantization:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\n# load in 8bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_8bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n\n# load in 4bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n### Expected speedups\n\nYou can benefit from considerable speedups for inference, especially for inputs with long sequences. However, since FlashAttention-2 does not support computing attention scores with padding tokens, you must manually pad/unpad the attention scores for batched inference when the sequence contains padding tokens. This leads to a significant slowdown for batched generations with padding tokens.\n\nTo overcome this, you should use FlashAttention-2 without padding tokens in the sequence during training (by packing a dataset or [concatenating sequences](https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py#L516) until reaching the maximum sequence length)."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: c38792b5-2aed-4e5c-841b-4ebff5c0222a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"c38792b5-2aed-4e5c-841b-4ebff5c0222a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e4443492-bb17-4fd9-9471-ff3515188706\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"91c701a7e0d46917003d34d930c9d69e6fb39271035231880ed8554373e59370\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b81548d4-9134-482c-869d-a1a8c81d110c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"021412bd0dd7de709897a2acfcfb1a710c3907684b4992499ef52384ec4955f5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7447, \"end_char_idx\": 10751, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference. For example, you can combine FlashAttention-2 with 8-bit or 4-bit quantization:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\n# load in 8bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_8bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n\n# load in 4bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n### Expected speedups\n\nYou can benefit from considerable speedups for inference, especially for inputs with long sequences. However, since FlashAttention-2 does not support computing attention scores with padding tokens, you must manually pad/unpad the attention scores for batched inference when the sequence contains padding tokens. This leads to a significant slowdown for batched generations with padding tokens.\n\nTo overcome this, you should use FlashAttention-2 without padding tokens in the sequence during training (by packing a dataset or [concatenating sequences](https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py#L516) until reaching the maximum sequence length).\n\nFor a single forward pass on [tiiuae/falcon-7b](https://hf.co/tiiuae/falcon-7b) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/falcon-7b-inference-large-seqlen.png\">\n</div>\n\nFor a single forward pass on [meta-llama/Llama-7b-hf](https://hf.co/meta-llama/Llama-7b-hf) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-7b-inference-large-seqlen.png\">\n</div>\n\nFor sequences with padding tokens (generating with padding tokens), you need to unpad/pad the input sequences to correctly compute the attention scores. With a relatively small sequence length, a single forward pass creates overhead leading to a small speedup (in the example below, 30% of the input is filled with padding tokens):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-small-seqlen-padding.png\">\n</div>\n\nBut for larger sequence lengths, you can expect even more speedup benefits:\n\n<Tip>\n\nFlashAttention is more memory efficient, meaning you can train on much larger sequence lengths without running into out-of-memory issues. You can potentially reduce memory usage up to 20x for larger sequence lengths. Take a look at the [flash-attention](https://github.com/Dao-AILab/flash-attention) repository for more details."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: b81548d4-9134-482c-869d-a1a8c81d110c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"b81548d4-9134-482c-869d-a1a8c81d110c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c38792b5-2aed-4e5c-841b-4ebff5c0222a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9302e1516b4bbdfa77cece7f5bf946f525393f1ce6bb0a67fbfc5fdc7bfc579b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8404729e-0266-42a7-9233-77dbb93c92aa\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"64b216d09d783aae60e7b812a523d8ea2808fcd2cf6a3bbeae6c6e3c4cea44e1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9134, \"end_char_idx\": 12234, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "For a single forward pass on [tiiuae/falcon-7b](https://hf.co/tiiuae/falcon-7b) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/falcon-7b-inference-large-seqlen.png\">\n</div>\n\nFor a single forward pass on [meta-llama/Llama-7b-hf](https://hf.co/meta-llama/Llama-7b-hf) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-7b-inference-large-seqlen.png\">\n</div>\n\nFor sequences with padding tokens (generating with padding tokens), you need to unpad/pad the input sequences to correctly compute the attention scores. With a relatively small sequence length, a single forward pass creates overhead leading to a small speedup (in the example below, 30% of the input is filled with padding tokens):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-small-seqlen-padding.png\">\n</div>\n\nBut for larger sequence lengths, you can expect even more speedup benefits:\n\n<Tip>\n\nFlashAttention is more memory efficient, meaning you can train on much larger sequence lengths without running into out-of-memory issues. You can potentially reduce memory usage up to 20x for larger sequence lengths. Take a look at the [flash-attention](https://github.com/Dao-AILab/flash-attention) repository for more details.\n\n</Tip>\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-large-seqlen-padding.png\">\n</div>\n\n## PyTorch scaled dot product attention\n\nPyTorch's [`torch.nn.functional.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA) can also call FlashAttention and memory-efficient attention kernels under the hood. SDPA support is currently being added natively in Transformers and is used by default for `torch>=2.1.1` when an implementation is available. You may also set `attn_implementation=\"sdpa\"` in `from_pretrained()` to explicitly request SDPA to be used.\n\nFor now, Transformers supports SDPA inference and training for the following architectures:\n* [Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer#transformers.ASTModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Bert](https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 8404729e-0266-42a7-9233-77dbb93c92aa
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8404729e-0266-42a7-9233-77dbb93c92aa\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b81548d4-9134-482c-869d-a1a8c81d110c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c2d983df83f80150ee4971d17e34ab9ac63707622a22846f6e5a73423efb92f7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d3be7135-b2a4-4046-89e8-1b6790e7e2bf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b437404cc5070611097c856ab6150cc7431f0682d86da6678dcca5994ad5b2af\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10753, \"end_char_idx\": 13432, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "</Tip>\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-large-seqlen-padding.png\">\n</div>\n\n## PyTorch scaled dot product attention\n\nPyTorch's [`torch.nn.functional.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA) can also call FlashAttention and memory-efficient attention kernels under the hood. SDPA support is currently being added natively in Transformers and is used by default for `torch>=2.1.1` when an implementation is available. You may also set `attn_implementation=\"sdpa\"` in `from_pretrained()` to explicitly request SDPA to be used.\n\nFor now, Transformers supports SDPA inference and training for the following architectures:\n* [Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer#transformers.ASTModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Bert](https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DeiT](https://huggingface.co/docs/transformers/model_doc/deit#transformers.DeiTModel)\n* [Dpr](https://huggingface.co/docs/transformers/model_doc/dpr#transformers.DprReader)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [PaliGemma](https://huggingface.co/docs/transformers/model_doc/paligemma#transformers."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: d3be7135-b2a4-4046-89e8-1b6790e7e2bf
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d3be7135-b2a4-4046-89e8-1b6790e7e2bf\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8404729e-0266-42a7-9233-77dbb93c92aa\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"fd7e77e16ac98b685a1432b2419acdbd39515804439a22d5d7ddc16731c7ff6c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2ea9bcbf-a265-484e-965f-06486204c007\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0552d18a9a088900be4334a6b20f9888679892a332ea562db622a6e2ca0d2dfc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12185, \"end_char_idx\": 14578, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DeiT](https://huggingface.co/docs/transformers/model_doc/deit#transformers.DeiTModel)\n* [Dpr](https://huggingface.co/docs/transformers/model_doc/dpr#transformers.DprReader)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [PaliGemma](https://huggingface.co/docs/transformers/model_doc/paligemma#transformers.PaliGemmaForConditionalGeneration)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [Idefics](https://huggingface.co/docs/transformers/model_doc/idefics#transformers.IdeficsModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2Audio](https://huggingface.co/docs/transformers/model_doc/qwen2_audio#transformers.Qwen2AudioEncoder)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Qwen2VL](https://huggingface."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 2ea9bcbf-a265-484e-965f-06486204c007
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2ea9bcbf-a265-484e-965f-06486204c007\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d3be7135-b2a4-4046-89e8-1b6790e7e2bf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1f075a8aa8b8ef814ba0aac07d6fff9d22d48d9eca10923f19d3f851ed713275\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ab864342-2b7a-4139-a7a2-8cab8d5b4d6b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a427202087ff194cc31264089bef6bf08f21b9a3e40bc93d14949a2653e8d944\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13378, \"end_char_idx\": 15773, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "co/docs/transformers/model_doc/paligemma#transformers.PaliGemmaForConditionalGeneration)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [Idefics](https://huggingface.co/docs/transformers/model_doc/idefics#transformers.IdeficsModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2Audio](https://huggingface.co/docs/transformers/model_doc/qwen2_audio#transformers.Qwen2AudioEncoder)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)\n* [ViT](https://huggingface.co/docs/transformers/model_doc/vit#transformers.ViTModel)\n* [ViTHybrid](https://huggingface.co/docs/transformers/model_doc/vit_hybrid#transformers.ViTHybridModel)\n* [ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae#transformers.ViTMAEModel)\n* [ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn#transformers.ViTMSNModel)\n* [VideoMAE](https://huggingface.co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell)\n* [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [SigLIP](https://huggingface."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: ab864342-2b7a-4139-a7a2-8cab8d5b4d6b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"ab864342-2b7a-4139-a7a2-8cab8d5b4d6b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2ea9bcbf-a265-484e-965f-06486204c007\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e4eff9ddbae31922d6f353973824646a5e71011e7f51981d3f6f15a91039a78f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7ac39afa-b729-4d7b-ad49-fbf7f8e820f6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c34720750aefc53f2a4dca785e4375a812cabbc86663d4d9c416554d5e05f1f4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14531, \"end_char_idx\": 16696, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "Qwen2MoeModel)\n* [Qwen2VL](https://huggingface.co/docs/transformers/model_doc/qwen2_vl#transformers.Qwen2VLModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [Nemotron](https://huggingface.co/docs/transformers/model_doc/nemotron)\n* [ViT](https://huggingface.co/docs/transformers/model_doc/vit#transformers.ViTModel)\n* [ViTHybrid](https://huggingface.co/docs/transformers/model_doc/vit_hybrid#transformers.ViTHybridModel)\n* [ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae#transformers.ViTMAEModel)\n* [ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn#transformers.ViTMSNModel)\n* [VideoMAE](https://huggingface.co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell)\n* [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n* [YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos#transformers.YolosModel)\n\n\n<Tip>\n\nFlashAttention can only be used for models with the `fp16` or `bf16` torch type, so make sure to cast your model to the appropriate type first. The memory-efficient attention backend is able to handle `fp32` models.\n\n</Tip>\n\n<Tip>\n\nSDPA does not support certain sets of attention parameters, such as `head_mask` and `output_attentions=True`.\nIn that case, you should see a warning message and we will fall back to the (slower) eager implementation."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 7ac39afa-b729-4d7b-ad49-fbf7f8e820f6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"7ac39afa-b729-4d7b-ad49-fbf7f8e820f6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ab864342-2b7a-4139-a7a2-8cab8d5b4d6b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"98ff72426e3260f5d0358244cf1e9779cabc027515ac8b74619f19bd5ebc102a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9e71fded-a859-469c-936a-1614ad709d5d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"64a4589c5365e2cffc9275a0dc5fb491bf9e06b34df55e674b8ae9408b016fe3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15357, \"end_char_idx\": 18270, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell)\n* [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n* [YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos#transformers.YolosModel)\n\n\n<Tip>\n\nFlashAttention can only be used for models with the `fp16` or `bf16` torch type, so make sure to cast your model to the appropriate type first. The memory-efficient attention backend is able to handle `fp32` models.\n\n</Tip>\n\n<Tip>\n\nSDPA does not support certain sets of attention parameters, such as `head_mask` and `output_attentions=True`.\nIn that case, you should see a warning message and we will fall back to the (slower) eager implementation.\n\n</Tip>\n\nBy default, SDPA selects the most performant kernel available but you can check whether a backend is available in a given setting (hardware, problem size) with [`torch.backends.cuda.sdp_kernel`](https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel) as a context manager:\n\n```diff\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.float16).to(\"cuda\")\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```\n\nIf you see a bug with the traceback below, try using the nightly version of PyTorch which may have broader coverage for FlashAttention:\n\n```bash\nRuntimeError: No available kernel. Aborting execution.\n\n# install PyTorch nightly\npip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118\n```\n\n## BetterTransformer\n\n<Tip warning={true}>\n\nSome BetterTransformer features are being upstreamed to Transformers with default support for native `torch.nn.scaled_dot_product_attention`. BetterTransformer still has a wider coverage than the Transformers SDPA integration, but you can expect more and more architectures to natively support SDPA in Transformers."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 9e71fded-a859-469c-936a-1614ad709d5d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9e71fded-a859-469c-936a-1614ad709d5d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7ac39afa-b729-4d7b-ad49-fbf7f8e820f6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9270e5562f160efcc3e2c7e13dc33af7b3457f91d1ddaa950ab987f893070525\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"75d49e06-3e23-4309-8478-bb714015f318\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"00c96c886c7f85e488f593363bf3b14d05feced924910e2dab5ec6cce564d6e7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16590, \"end_char_idx\": 20137, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "In that case, you should see a warning message and we will fall back to the (slower) eager implementation.\n\n</Tip>\n\nBy default, SDPA selects the most performant kernel available but you can check whether a backend is available in a given setting (hardware, problem size) with [`torch.backends.cuda.sdp_kernel`](https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel) as a context manager:\n\n```diff\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.float16).to(\"cuda\")\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```\n\nIf you see a bug with the traceback below, try using the nightly version of PyTorch which may have broader coverage for FlashAttention:\n\n```bash\nRuntimeError: No available kernel. Aborting execution.\n\n# install PyTorch nightly\npip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118\n```\n\n## BetterTransformer\n\n<Tip warning={true}>\n\nSome BetterTransformer features are being upstreamed to Transformers with default support for native `torch.nn.scaled_dot_product_attention`. BetterTransformer still has a wider coverage than the Transformers SDPA integration, but you can expect more and more architectures to natively support SDPA in Transformers.\n\n</Tip>\n\n<Tip>\n\nCheck out our benchmarks with BetterTransformer and scaled dot product attention in the [Out of the box acceleration and memory savings of \ud83e\udd17 decoder models with PyTorch 2.0](https://pytorch.org/blog/out-of-the-box-acceleration/) and learn more about the fastpath execution in the [BetterTransformer](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2) blog post.\n\n</Tip>\n\nBetterTransformer accelerates inference with its fastpath (native PyTorch specialized implementation of Transformer functions) execution. The two optimizations in the fastpath execution are:\n\n1. fusion, which combines multiple sequential operations into a single \"kernel\" to reduce the number of computation steps\n2. skipping the inherent sparsity of padding tokens to avoid unnecessary computation with nested tensors\n\nBetterTransformer also converts all attention operations to use the more memory-efficient [scaled dot product attention (SDPA)](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention), and it calls optimized kernels like [FlashAttention](https://huggingface.co/papers/2205.14135) under the hood.\n\nBefore you start, make sure you have \ud83e\udd17 Optimum [installed](https://huggingface.co/docs/optimum/installation).\n\nThen you can enable BetterTransformer with the [`PreTrainedModel.to_bettertransformer`] method:\n\n```python\nmodel = model.to_bettertransformer()\n```\n\nYou can return the original Transformers model with the [`~PreTrainedModel.reverse_bettertransformer`] method. You should use this before saving your model to use the canonical Transformers modeling:\n\n```py\nmodel = model.reverse_bettertransformer()\nmodel.save_pretrained(\"saved_model\")\n```\n\n## bitsandbytes\n\nbitsandbytes is a quantization library that includes support for 4-bit and 8-bit quantization."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 75d49e06-3e23-4309-8478-bb714015f318
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"75d49e06-3e23-4309-8478-bb714015f318\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9e71fded-a859-469c-936a-1614ad709d5d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"75789e601d8fed66cd86c7e60eaa01c6f50f67aa3561e6e08950985b1c823df7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3c220fac-6cdb-44e1-b919-cb290625ebef\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"15a05defd441bde2c76c492113669685e575ad50b4b4605dc3b7377dc88771ee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18714, \"end_char_idx\": 21859, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "</Tip>\n\nBetterTransformer accelerates inference with its fastpath (native PyTorch specialized implementation of Transformer functions) execution. The two optimizations in the fastpath execution are:\n\n1. fusion, which combines multiple sequential operations into a single \"kernel\" to reduce the number of computation steps\n2. skipping the inherent sparsity of padding tokens to avoid unnecessary computation with nested tensors\n\nBetterTransformer also converts all attention operations to use the more memory-efficient [scaled dot product attention (SDPA)](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention), and it calls optimized kernels like [FlashAttention](https://huggingface.co/papers/2205.14135) under the hood.\n\nBefore you start, make sure you have \ud83e\udd17 Optimum [installed](https://huggingface.co/docs/optimum/installation).\n\nThen you can enable BetterTransformer with the [`PreTrainedModel.to_bettertransformer`] method:\n\n```python\nmodel = model.to_bettertransformer()\n```\n\nYou can return the original Transformers model with the [`~PreTrainedModel.reverse_bettertransformer`] method. You should use this before saving your model to use the canonical Transformers modeling:\n\n```py\nmodel = model.reverse_bettertransformer()\nmodel.save_pretrained(\"saved_model\")\n```\n\n## bitsandbytes\n\nbitsandbytes is a quantization library that includes support for 4-bit and 8-bit quantization. Quantization reduces your model size compared to its native full precision version, making it easier to fit large models onto GPUs with limited memory.\n\nMake sure you have bitsandbytes and \ud83e\udd17 Accelerate installed:\n\n```bash\n# these versions support 8-bit and 4-bit\npip install bitsandbytes>=0.39.0 accelerate>=0.20.0\n\n# install Transformers\npip install transformers\n```\n\n### 4-bit\n\nTo load a model in 4-bit for inference, use the `load_in_4bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"auto\", load_in_4bit=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 600MB of memory to the first GPU and 1GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"600MB\", 1: \"1GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_4bit=True, max_memory=max_memory_mapping\n)\n```\n\n### 8-bit\n\n<Tip>\n\nIf you're curious and interested in learning more about the concepts underlying 8-bit quantization, read the [Gentle Introduction to 8-bit Matrix Multiplication for transformers at scale using Hugging Face Transformers, Accelerate and bitsandbytes](https://huggingface.co/blog/hf-bitsandbytes-integration) blog post.\n\n</Tip>\n\nTo load a model in 8-bit for inference, use the `load_in_8bit` parameter."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 3c220fac-6cdb-44e1-b919-cb290625ebef
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3c220fac-6cdb-44e1-b919-cb290625ebef\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"75d49e06-3e23-4309-8478-bb714015f318\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f820deba41631c0222895c433e3a10261ac898e707d1fc3c372615e9ad6a8856\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b4a3115e-8456-4809-8b28-f703c5f3fe92\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"39d7cf5d1af214a00d8b78dfe1c506987d0bbc3c4b5d3287bd66e8c22eed2595\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20592, \"end_char_idx\": 23338, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"auto\", load_in_4bit=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 600MB of memory to the first GPU and 1GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"600MB\", 1: \"1GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_4bit=True, max_memory=max_memory_mapping\n)\n```\n\n### 8-bit\n\n<Tip>\n\nIf you're curious and interested in learning more about the concepts underlying 8-bit quantization, read the [Gentle Introduction to 8-bit Matrix Multiplication for transformers at scale using Hugging Face Transformers, Accelerate and bitsandbytes](https://huggingface.co/blog/hf-bitsandbytes-integration) blog post.\n\n</Tip>\n\nTo load a model in 8-bit for inference, use the `load_in_8bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment:\n\n```py\nfrom transformers import AutoModelForCausalLM, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n```\n\nIf you're loading a model in 8-bit for text generation, you should use the [`~transformers.GenerationMixin.generate`] method instead of the [`Pipeline`] function which is not optimized for 8-bit models and will be slower. Some sampling strategies, like nucleus sampling, are also not supported by the [`Pipeline`] for 8-bit models. You should also place all inputs on the same device as the model:\n\n```py\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n\nprompt = \"Hello, my llama is cute\"\ninputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\ngenerated_ids = model.generate(**inputs)\noutputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: b4a3115e-8456-4809-8b28-f703c5f3fe92
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"b4a3115e-8456-4809-8b28-f703c5f3fe92\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c220fac-6cdb-44e1-b919-cb290625ebef\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"52d34ac199e41d736525b26a537e0fefba4f4cef3ba06c5e6ec8861c67715901\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6a48f73c-90df-430c-bd07-be394955b3b4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"895f2edd7d2c13665ce49c60128285f3796008c3f1afb6469f60dda0c892ac13\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21778, \"end_char_idx\": 24614, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "</Tip>\n\nTo load a model in 8-bit for inference, use the `load_in_8bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment:\n\n```py\nfrom transformers import AutoModelForCausalLM, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n```\n\nIf you're loading a model in 8-bit for text generation, you should use the [`~transformers.GenerationMixin.generate`] method instead of the [`Pipeline`] function which is not optimized for 8-bit models and will be slower. Some sampling strategies, like nucleus sampling, are also not supported by the [`Pipeline`] for 8-bit models. You should also place all inputs on the same device as the model:\n\n```py\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n\nprompt = \"Hello, my llama is cute\"\ninputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\ngenerated_ids = model.generate(**inputs)\noutputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 1GB of memory to the first GPU and 2GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"1GB\", 1: \"2GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_8bit=True, max_memory=max_memory_mapping\n)\n```\n\n<Tip>\n\nFeel free to try running a 11 billion parameter [T5 model](https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing) or the 3 billion parameter [BLOOM model](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing) for inference on Google Colab's free tier GPUs!\n\n</Tip>\n\n## \ud83e\udd17 Optimum\n\n<Tip>\n\nLearn more details about using ORT with \ud83e\udd17 Optimum in the [Accelerated inference on NVIDIA GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides. This section only provides a brief and simple example.\n\n</Tip>\n\nONNX Runtime (ORT) is a model accelerator that supports accelerated inference on Nvidia GPUs, and AMD GPUs that use [ROCm](https://www.amd.com/en/products/software/rocm.html) stack."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 6a48f73c-90df-430c-bd07-be394955b3b4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"6a48f73c-90df-430c-bd07-be394955b3b4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b4a3115e-8456-4809-8b28-f703c5f3fe92\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f61df08f36906633c1136352d7f66ea6335e3216e6a3ce5e24c28e56bae17b6f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"775ee1e1-4f94-4dfc-8bd8-fbccf5e87241\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4fb7fa09c09665776357c1f165b687645b85ad0ae4e9cc8bc300cb83ee1c2ff0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23339, \"end_char_idx\": 25503, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "For example, to distribute 1GB of memory to the first GPU and 2GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"1GB\", 1: \"2GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_8bit=True, max_memory=max_memory_mapping\n)\n```\n\n<Tip>\n\nFeel free to try running a 11 billion parameter [T5 model](https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing) or the 3 billion parameter [BLOOM model](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing) for inference on Google Colab's free tier GPUs!\n\n</Tip>\n\n## \ud83e\udd17 Optimum\n\n<Tip>\n\nLearn more details about using ORT with \ud83e\udd17 Optimum in the [Accelerated inference on NVIDIA GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides. This section only provides a brief and simple example.\n\n</Tip>\n\nONNX Runtime (ORT) is a model accelerator that supports accelerated inference on Nvidia GPUs, and AMD GPUs that use [ROCm](https://www.amd.com/en/products/software/rocm.html) stack. ORT uses optimization techniques like fusing common operations into a single node and constant folding to reduce the number of computations performed and speedup inference. ORT also places the most computationally intensive operations on the GPU and the rest on the CPU to intelligently distribute the workload between the two devices.\n\nORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider)."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 775ee1e1-4f94-4dfc-8bd8-fbccf5e87241
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"775ee1e1-4f94-4dfc-8bd8-fbccf5e87241\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6a48f73c-90df-430c-bd07-be394955b3b4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a22ee9204526c55a30f35d69b52e26ac97aad75cecebfa8531e54d19a707b61a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dcefcf4c-2950-47e8-972e-4d69342131ec\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6983403900b43b4cde8e5634c3937a5cb5a14610c1a42b3f450fc8fa4bf4251a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23990, \"end_char_idx\": 26530, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "</Tip>\n\n## \ud83e\udd17 Optimum\n\n<Tip>\n\nLearn more details about using ORT with \ud83e\udd17 Optimum in the [Accelerated inference on NVIDIA GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides. This section only provides a brief and simple example.\n\n</Tip>\n\nONNX Runtime (ORT) is a model accelerator that supports accelerated inference on Nvidia GPUs, and AMD GPUs that use [ROCm](https://www.amd.com/en/products/software/rocm.html) stack. ORT uses optimization techniques like fusing common operations into a single node and constant folding to reduce the number of computations performed and speedup inference. ORT also places the most computationally intensive operations on the GPU and the rest on the CPU to intelligently distribute the workload between the two devices.\n\nORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider). If you want to load a model that was not yet exported to ONNX, you can set `export=True` to convert your model on-the-fly to the ONNX format:\n\n```py\nfrom optimum.onnxruntime import ORTModelForSequenceClassification\n\nort_model = ORTModelForSequenceClassification.from_pretrained(\n  \"distilbert/distilbert-base-uncased-finetuned-sst-2-english\",\n  export=True,\n  provider=\"CUDAExecutionProvider\",\n)\n```\n\nNow you're free to use the model for inference:\n\n```py\nfrom optimum.pipelines import pipeline\nfrom transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased-finetuned-sst-2-english\")\n\npipeline = pipeline(task=\"text-classification\", model=ort_model, tokenizer=tokenizer, device=\"cuda:0\")\nresult = pipeline(\"Both the music and visual were astounding, not to mention the actors performance.\")\n```\n\n## Combine optimizations\n\nIt is often possible to combine several of the optimization techniques described above to get the best inference performance possible for your model."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: dcefcf4c-2950-47e8-972e-4d69342131ec
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7227, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"dcefcf4c-2950-47e8-972e-4d69342131ec\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3af22dbf-d566-4ef7-8034-84cbf2a07850\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9ab6187133ff4cf61b86f445d5d8fef44bd89a0ad3ec7e01ff7149edf99f2303\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"775ee1e1-4f94-4dfc-8bd8-fbccf5e87241\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7227, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1922a9164bf87289112b121c2dc94f4306114d518b846d897ab25ca5a96b2fe1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24952, \"end_char_idx\": 27435, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "ref_doc_id": "3af22dbf-d566-4ef7-8034-84cbf2a07850", "chroma:document": "ORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider). If you want to load a model that was not yet exported to ONNX, you can set `export=True` to convert your model on-the-fly to the ONNX format:\n\n```py\nfrom optimum.onnxruntime import ORTModelForSequenceClassification\n\nort_model = ORTModelForSequenceClassification.from_pretrained(\n  \"distilbert/distilbert-base-uncased-finetuned-sst-2-english\",\n  export=True,\n  provider=\"CUDAExecutionProvider\",\n)\n```\n\nNow you're free to use the model for inference:\n\n```py\nfrom optimum.pipelines import pipeline\nfrom transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased-finetuned-sst-2-english\")\n\npipeline = pipeline(task=\"text-classification\", model=ort_model, tokenizer=tokenizer, device=\"cuda:0\")\nresult = pipeline(\"Both the music and visual were astounding, not to mention the actors performance.\")\n```\n\n## Combine optimizations\n\nIt is often possible to combine several of the optimization techniques described above to get the best inference performance possible for your model. For example, you can load a model in 4-bit, and then enable BetterTransformer with FlashAttention:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\n# load model in 4-bit\nquantization_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    bnb_4bit_compute_dtype=torch.float16\n)\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", quantization_config=quantization_config)\n\n# enable BetterTransformer\nmodel = model.to_bettertransformer()\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n# enable FlashAttention\nwith torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 5646e085-fb72-44dd-ae7b-df286d64f7fc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5646e085-fb72-44dd-ae7b-df286d64f7fc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2227587a-0150-46ef-8eee-7c3a09ef77b9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9ccfbd280615fea0c7e9ac62bd7ee5551db95b730ecddcfac28f222f8a580983\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2738, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "# Efficient Training on Multiple CPUs\n\nWhen training on a single CPU is too slow, we can use multiple CPUs. This guide focuses on PyTorch-based DDP enabling\ndistributed CPU training efficiently on [bare metal](#usage-in-trainer) and [Kubernetes](#usage-with-kubernetes).\n\n## Intel\u00ae oneCCL Bindings for PyTorch\n\n[Intel\u00ae oneCCL](https://github.com/oneapi-src/oneCCL) (collective communications library) is a library for efficient distributed deep learning training implementing such collectives like allreduce, allgather, alltoall. For more information on oneCCL, please refer to the [oneCCL documentation](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html) and [oneCCL specification](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html).\n\nModule `oneccl_bindings_for_pytorch` (`torch_ccl` before version 1.12)  implements PyTorch C10D ProcessGroup API and can be dynamically loaded as external ProcessGroup and only works on Linux platform now\n\nCheck more detailed information for [oneccl_bind_pt](https://github.com/intel/torch-ccl).\n\n### Intel\u00ae oneCCL Bindings for PyTorch installation\n\nWheel files are available for the following Python versions:\n\n| Extension Version | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 |\n| :---------------: | :--------: | :--------: | :--------: | :--------: | :---------: |\n| 2.1.0             |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 2.0.0             |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.13.0            |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.12.100          |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.12.0            |            | \u221a          | \u221a          | \u221a          | \u221a           |\n\nPlease run `pip list | grep torch` to get your `pytorch_version`.\n```bash\npip install oneccl_bind_pt=={pytorch_version} -f https://developer.intel.com/ipex-whl-stable-cpu\n```\nwhere `{pytorch_version}` should be your PyTorch version, for instance 2.1.0.\nCheck more approaches for [oneccl_bind_pt installation](https://github.com/intel/torch-ccl).\nVersions of oneCCL and PyTorch must match.\n\n<Tip warning={true}>\n\noneccl_bindings_for_pytorch 1.12.0 prebuilt wheel does not work with PyTorch 1.12.1 (it is for PyTorch 1.12.0)\nPyTorch 1.12.1 should work with oneccl_bindings_for_pytorch 1.12.100\n\n</Tip>\n\n## Intel\u00ae MPI library\nUse this standards-based MPI implementation to deliver flexible, efficient, scalable cluster messaging on Intel\u00ae architecture. This component is part of the Intel\u00ae oneAPI HPC Toolkit.\n\noneccl_bindings_for_pytorch is installed along with the MPI tool set. Need to source the environment before using it."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 2227587a-0150-46ef-8eee-7c3a09ef77b9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2227587a-0150-46ef-8eee-7c3a09ef77b9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5646e085-fb72-44dd-ae7b-df286d64f7fc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"bafb8174e88bbfddc6cadb16953124a197a232605fb00ba122b998a09be94dae\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7e5c2e21-d17e-4b55-a85d-11fd4bbfe25c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cc42a1766cb4ac5780371b8b51f5518e1f7b69ad5217ba383aec1bfae4ddac46\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1879, \"end_char_idx\": 4762, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "```bash\npip install oneccl_bind_pt=={pytorch_version} -f https://developer.intel.com/ipex-whl-stable-cpu\n```\nwhere `{pytorch_version}` should be your PyTorch version, for instance 2.1.0.\nCheck more approaches for [oneccl_bind_pt installation](https://github.com/intel/torch-ccl).\nVersions of oneCCL and PyTorch must match.\n\n<Tip warning={true}>\n\noneccl_bindings_for_pytorch 1.12.0 prebuilt wheel does not work with PyTorch 1.12.1 (it is for PyTorch 1.12.0)\nPyTorch 1.12.1 should work with oneccl_bindings_for_pytorch 1.12.100\n\n</Tip>\n\n## Intel\u00ae MPI library\nUse this standards-based MPI implementation to deliver flexible, efficient, scalable cluster messaging on Intel\u00ae architecture. This component is part of the Intel\u00ae oneAPI HPC Toolkit.\n\noneccl_bindings_for_pytorch is installed along with the MPI tool set. Need to source the environment before using it.\n\nfor Intel\u00ae oneCCL >= 1.12.0\n```bash\noneccl_bindings_for_pytorch_path=$(python -c \"from oneccl_bindings_for_pytorch import cwd; print(cwd)\")\nsource $oneccl_bindings_for_pytorch_path/env/setvars.sh\n```\n\nfor Intel\u00ae oneCCL whose version < 1.12.0\n```bash\ntorch_ccl_path=$(python -c \"import torch; import torch_ccl; import os;  print(os.path.abspath(os.path.dirname(torch_ccl.__file__)))\")\nsource $torch_ccl_path/env/setvars.sh\n```\n\n#### Intel\u00ae Extension for PyTorch installation\n\nIntel Extension for PyTorch (IPEX) provides performance optimizations for CPU training with both Float32 and BFloat16 (refer to the [single CPU section](./perf_train_cpu) to learn more).\n\n\nThe following \"Usage in Trainer\" takes mpirun in Intel\u00ae MPI library as an example.\n\n\n## Usage in Trainer\nTo enable multi CPU distributed training in the Trainer with the ccl backend, users should add **`--ddp_backend ccl`** in the command arguments.\n\nLet's see an example with the [question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)\n\n\nThe following command enables training with 2 processes on one Xeon node, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=127.0.0.1\n mpirun -n 2 -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex\n```\nThe following command enables training with a total of four processes on two Xeons (node0 and node1, taking node0 as the main process), ppn (processes per node) is set to 2, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 7e5c2e21-d17e-4b55-a85d-11fd4bbfe25c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"7e5c2e21-d17e-4b55-a85d-11fd4bbfe25c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2227587a-0150-46ef-8eee-7c3a09ef77b9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d340ba85ea319c283961145e611f9782806746e9e14948582422ab305ed99fd5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4e5dc914-578b-4a4d-868a-c894ebf19509\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"68ae3027c0bb5c2d3a60e6ab6c18dd5847021e92b8559ebe603852bb4b0cba5b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3404, \"end_char_idx\": 6320, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "The following \"Usage in Trainer\" takes mpirun in Intel\u00ae MPI library as an example.\n\n\n## Usage in Trainer\nTo enable multi CPU distributed training in the Trainer with the ccl backend, users should add **`--ddp_backend ccl`** in the command arguments.\n\nLet's see an example with the [question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)\n\n\nThe following command enables training with 2 processes on one Xeon node, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=127.0.0.1\n mpirun -n 2 -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex\n```\nThe following command enables training with a total of four processes on two Xeons (node0 and node1, taking node0 as the main process), ppn (processes per node) is set to 2, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n\nIn node0, you need to create a configuration file which contains the IP addresses of each node (for example hostfile) and pass that configuration file path as an argument.\n```shell script\n cat hostfile\n xxx.xxx.xxx.xxx #node0 ip\n xxx.xxx.xxx.xxx #node1 ip\n```\nNow, run the following command in node0 and **4DDP** will be enabled in node0 and node1 with BF16 auto mixed precision:\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=xxx.xxx.xxx.xxx #node0 ip\n mpirun -f hostfile -n 4 -ppn 2 \\\n -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex \\\n --bf16\n```\n\n## Usage with Kubernetes\n\nThe same distributed training job from the previous section can be deployed to a Kubernetes cluster using the\n[Kubeflow PyTorchJob training operator](https://www.kubeflow.org/docs/components/training/pytorch/).\n\n### Setup\n\nThis example assumes that you have:\n* Access to a Kubernetes cluster with [Kubeflow installed](https://www.kubeflow.org/docs/started/installing-kubeflow/)\n* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) installed and configured to access the Kubernetes cluster\n* A [Persistent Volume Claim (PVC)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) that can be used\n  to store datasets and model files."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 4e5dc914-578b-4a4d-868a-c894ebf19509
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4e5dc914-578b-4a4d-868a-c894ebf19509\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7e5c2e21-d17e-4b55-a85d-11fd4bbfe25c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"6abd131e8d06e12a79d46291d1e7f9a32e0551911a560f0875c23652d74a07bf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"67c71e14-f6d8-4ebe-a788-b022288c8ab9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"91f55f0a24902c6cfe93427e3094539cf298635f5dc3b45065e20bcddd7ac1d6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4936, \"end_char_idx\": 7955, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "```shell script\n cat hostfile\n xxx.xxx.xxx.xxx #node0 ip\n xxx.xxx.xxx.xxx #node1 ip\n```\nNow, run the following command in node0 and **4DDP** will be enabled in node0 and node1 with BF16 auto mixed precision:\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=xxx.xxx.xxx.xxx #node0 ip\n mpirun -f hostfile -n 4 -ppn 2 \\\n -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex \\\n --bf16\n```\n\n## Usage with Kubernetes\n\nThe same distributed training job from the previous section can be deployed to a Kubernetes cluster using the\n[Kubeflow PyTorchJob training operator](https://www.kubeflow.org/docs/components/training/pytorch/).\n\n### Setup\n\nThis example assumes that you have:\n* Access to a Kubernetes cluster with [Kubeflow installed](https://www.kubeflow.org/docs/started/installing-kubeflow/)\n* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) installed and configured to access the Kubernetes cluster\n* A [Persistent Volume Claim (PVC)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) that can be used\n  to store datasets and model files. There are multiple options for setting up the PVC including using an NFS\n  [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/) or a cloud storage bucket.\n* A Docker container that includes your model training script and all the dependencies needed to run the script. For\n  distributed CPU training jobs, this typically includes PyTorch, Transformers, Intel Extension for PyTorch, Intel\n  oneCCL Bindings for PyTorch, and OpenSSH to communicate between the containers.\n\nThe snippet below is an example of a Dockerfile that uses a base image that supports distributed CPU training and then\nextracts a Transformers release to the `/workspace` directory, so that the example scripts are included in the image:\n```dockerfile\nFROM intel/intel-optimized-pytorch:2.3.0-pip-multinode\n\nRUN apt-get update -y && \\\n    apt-get install -y --no-install-recommends --fix-missing \\\n    google-perftools \\\n    libomp-dev\n\nWORKDIR /workspace\n\n# Download and extract the transformers code\nARG HF_TRANSFORMERS_VER=\"4.44.0\"\nRUN pip install --no-cache-dir \\\n    transformers==${HF_TRANSFORMERS_VER} && \\\n    mkdir transformers && \\\n    curl -sSL --retry 5 https://github.com/huggingface/transformers/archive/refs/tags/v${HF_TRANSFORMERS_VER}.tar.gz | tar -C transformers --strip-components=1 -xzf -\n```\nThe image needs to be built and copied to the cluster's nodes or pushed to a container registry prior to deploying the\nPyTorchJob to the cluster.\n\n### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 67c71e14-f6d8-4ebe-a788-b022288c8ab9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"67c71e14-f6d8-4ebe-a788-b022288c8ab9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4e5dc914-578b-4a4d-868a-c894ebf19509\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f282682efa4e18efe3b23c8b97aa85e717cd31609cf6ae7b232cfe2e412255ff\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8780f8a1-0d20-4315-9872-01e7e8e01915\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3709aaeca6664d421173c5be7b3e25b8ef305d519fb011cf775ee66345c3c80b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6321, \"end_char_idx\": 8772, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "There are multiple options for setting up the PVC including using an NFS\n  [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/) or a cloud storage bucket.\n* A Docker container that includes your model training script and all the dependencies needed to run the script. For\n  distributed CPU training jobs, this typically includes PyTorch, Transformers, Intel Extension for PyTorch, Intel\n  oneCCL Bindings for PyTorch, and OpenSSH to communicate between the containers.\n\nThe snippet below is an example of a Dockerfile that uses a base image that supports distributed CPU training and then\nextracts a Transformers release to the `/workspace` directory, so that the example scripts are included in the image:\n```dockerfile\nFROM intel/intel-optimized-pytorch:2.3.0-pip-multinode\n\nRUN apt-get update -y && \\\n    apt-get install -y --no-install-recommends --fix-missing \\\n    google-perftools \\\n    libomp-dev\n\nWORKDIR /workspace\n\n# Download and extract the transformers code\nARG HF_TRANSFORMERS_VER=\"4.44.0\"\nRUN pip install --no-cache-dir \\\n    transformers==${HF_TRANSFORMERS_VER} && \\\n    mkdir transformers && \\\n    curl -sSL --retry 5 https://github.com/huggingface/transformers/archive/refs/tags/v${HF_TRANSFORMERS_VER}.tar.gz | tar -C transformers --strip-components=1 -xzf -\n```\nThe image needs to be built and copied to the cluster's nodes or pushed to a container registry prior to deploying the\nPyTorchJob to the cluster.\n\n### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster. The yaml file for the PyTorchJob defines parameters such as:\n * The name of the PyTorchJob\n * The number of replicas (workers)\n * The python script and it's parameters that will be used to run the training job\n * The types of resources (node selector, memory, and CPU) needed for each worker\n * The image/tag for the Docker container to use\n * Environment variables\n * A volume mount for the PVC\n\nThe volume mount defines a path where the PVC will be mounted in the container for each worker pod. This location can be\nused for the dataset, checkpoint files, and the saved model after training completes.\n\nThe snippet below is an example of a yaml file for a PyTorchJob with 4 workers running the\n[question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 8780f8a1-0d20-4315-9872-01e7e8e01915
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8780f8a1-0d20-4315-9872-01e7e8e01915\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"67c71e14-f6d8-4ebe-a788-b022288c8ab9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"054b3a0a061118addeadff1caf8a002ca3383d9a70aba7bcc32961d0a56aa9a8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1ee5e36d-46ea-4697-b79e-27537d50ce33\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3a828dc821d444a53b87001c10311bad0a1a5748a89fc0144c9c46c4ac7b285e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7775, \"end_char_idx\": 11907, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster. The yaml file for the PyTorchJob defines parameters such as:\n * The name of the PyTorchJob\n * The number of replicas (workers)\n * The python script and it's parameters that will be used to run the training job\n * The types of resources (node selector, memory, and CPU) needed for each worker\n * The image/tag for the Docker container to use\n * Environment variables\n * A volume mount for the PVC\n\nThe volume mount defines a path where the PVC will be mounted in the container for each worker pod. This location can be\nused for the dataset, checkpoint files, and the saved model after training completes.\n\nThe snippet below is an example of a yaml file for a PyTorchJob with 4 workers running the\n[question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering).\n```yaml\napiVersion: \"kubeflow.org/v1\"\nkind: PyTorchJob\nmetadata:\n  name: transformers-pytorchjob\nspec:\n  elasticPolicy:\n    rdzvBackend: c10d\n    minReplicas: 1\n    maxReplicas: 4\n    maxRestarts: 10\n  pytorchReplicaSpecs:\n    Worker:\n      replicas: 4  # The number of worker pods\n      restartPolicy: OnFailure\n      template:\n        spec:\n          containers:\n            - name: pytorch\n              image: <image name>:<tag>  # Specify the docker image to use for the worker pods\n              imagePullPolicy: IfNotPresent\n              command: [\"/bin/bash\", \"-c\"]\n              args:\n                - >-\n                  cd /workspace/transformers;\n                  pip install -r /workspace/transformers/examples/pytorch/question-answering/requirements.txt;\n                  source /usr/local/lib/python3.10/dist-packages/oneccl_bindings_for_pytorch/env/setvars.sh;\n                  torchrun /workspace/transformers/examples/pytorch/question-answering/run_qa.py \\\n                    --model_name_or_path distilbert/distilbert-base-uncased \\\n                    --dataset_name squad \\\n                    --do_train \\\n                    --do_eval \\\n                    --per_device_train_batch_size 12 \\\n                    --learning_rate 3e-5 \\\n                    --num_train_epochs 2 \\\n                    --max_seq_length 384 \\\n                    --doc_stride 128 \\\n                    --output_dir /tmp/pvc-mount/output_$(date +%Y%m%d_%H%M%S) \\\n                    --no_cuda \\\n                    --ddp_backend ccl \\\n                    --bf16 \\\n                    --use_ipex;\n              env:\n              - name: LD_PRELOAD\n                value: \"/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4.5.9:/usr/local/lib/libiomp5.so\"\n              - name: TRANSFORMERS_CACHE\n                value: \"/tmp/pvc-mount/transformers_cache\"\n              - name: HF_DATASETS_CACHE\n                value: \"/tmp/pvc-mount/hf_datasets_cache\"\n              - name: LOGLEVEL\n                value: \"INFO\"\n              - name: CCL_WORKER_COUNT\n                value: \"1\"\n              - name: OMP_NUM_THREADS  # Can be tuned for optimal performance\n                value: \"240\"\n              resources:\n                limits:\n                  cpu: 240  # Update the CPU and memory limit values based on your nodes\n                  memory: 128Gi\n                requests:\n                  cpu: 240  # Update the CPU and memory request values based on your nodes\n                  memory: 128Gi\n              volumeMounts:\n              - name: pvc-volume\n                mountPath: /tmp/pvc-mount\n              - mountPath: /dev/shm\n                name: dshm\n          restartPolicy: Never\n          nodeSelector:  # Optionally use nodeSelector to match a certain node label for the worker pods\n            node-type: gnr\n          volumes:\n          - name: pvc-volume\n            persistentVolumeClaim:\n              claimName: transformers-pvc\n          - name: dshm\n            emptyDir:\n              medium: Memory\n```\nTo run this example, update the yaml based on your training script and the nodes in your cluster."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 1ee5e36d-46ea-4697-b79e-27537d50ce33
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3745, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"1ee5e36d-46ea-4697-b79e-27537d50ce33\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a976eb2-9650-4638-be27-0840e80eb0c4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"90bf510721ec54ac392e192edee32d70442722de8c43e41a564ce98d29812cda\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8780f8a1-0d20-4315-9872-01e7e8e01915\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3745, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"aa268a565fb3e1940a61c23bb6f5da12e12bdb419aaad3c830542560b94a8e8e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11909, \"end_char_idx\": 14737, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "ref_doc_id": "0a976eb2-9650-4638-be27-0840e80eb0c4", "chroma:document": "<Tip>\n\nThe CPU resource limits/requests in the yaml are defined in [cpu units](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-cpu)\nwhere 1 CPU unit is equivalent to 1 physical CPU core or 1 virtual core (depending on whether the node is a physical\nhost or a VM). The amount of CPU and memory limits/requests defined in the yaml should be less than the amount of\navailable CPU/memory capacity on a single machine. It is usually a good idea to not use the entire machine's capacity in\norder to leave some resources for the kubelet and OS. In order to get [\"guaranteed\"](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#guaranteed)\n[quality of service](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/) for the worker pods,\nset the same CPU and memory amounts for both the resource limits and requests.\n\n</Tip>\n\n### Deploy\n\nAfter the PyTorchJob spec has been updated with values appropriate for your cluster and training job, it can be deployed\nto the cluster using:\n```bash\nexport NAMESPACE=<specify your namespace>\n\nkubectl create -f pytorchjob.yaml -n ${NAMESPACE}\n```\n\nThe `kubectl get pods -n ${NAMESPACE}` command can then be used to list the pods in your namespace. You should see\nthe worker pods for the PyTorchJob that was just deployed. At first, they will probably have a status of \"Pending\" as\nthe containers get pulled and created, then the status should change to \"Running\".\n```\nNAME                                                     READY   STATUS                  RESTARTS          AGE\n...\ntransformers-pytorchjob-worker-0                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-1                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-2                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-3                         1/1     Running                 0                 7m37s\n...\n```\n\nThe logs for worker can be viewed using `kubectl logs <pod name> -n ${NAMESPACE}`. Add `-f` to stream the logs, for example:\n```bash\nkubectl logs transformers-pytorchjob-worker-0 -n ${NAMESPACE} -f\n```\n\nAfter the training job completes, the trained model can be copied from the PVC or storage location. When you are done\nwith the job, the PyTorchJob resource can be deleted from the cluster using `kubectl delete -f pytorchjob.yaml -n ${NAMESPACE}`.\n\n## Summary\n\nThis guide covered running distributed PyTorch training jobs using multiple CPUs on bare metal and on a Kubernetes\ncluster. Both cases utilize Intel Extension for PyTorch and Intel oneCCL Bindings for PyTorch for optimal training\nperformance, and can be used as a template to run your own workload on multiple nodes."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: b574880f-18f4-47cd-b3ad-45061fb60e8a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/pad_truncation", "title": "Padding and truncation", "tokens": 1364, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"b574880f-18f4-47cd-b3ad-45061fb60e8a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c3429a14-958d-4f31-8c2b-a32ff4a1a597\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"fcbf39d53d83c6f064cccf34a7654672d76b41edb3c0d340b128169d06afb319\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"18b8e036-a6e7-4d49-8d33-636687d3bf73\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e470205c927e63b5257be0726a9e40a2cb53fc05f060c8e73d6a930dc7d75c45\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3168, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "doc_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "ref_doc_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "chroma:document": "# Padding and truncation\n\nBatched inputs are often different lengths, so they can't be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special **padding token** to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences.\n\nIn most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: `padding`, `truncation` and `max_length`.\n\nThe `padding` argument controls padding. It can be a boolean or a string:\n\n  - `True` or `'longest'`: pad to the longest sequence in the batch (no padding is applied if you only provide\n    a single sequence).\n  - `'max_length'`: pad to a length specified by the `max_length` argument or the maximum length accepted\n    by the model if no `max_length` is provided (`max_length=None`). Padding will still be applied if you only provide a single sequence.\n  - `False` or `'do_not_pad'`: no padding is applied. This is the default behavior.\n\nThe `truncation` argument controls truncation. It can be a boolean or a string:\n\n  - `True` or `'longest_first'`: truncate to a maximum length specified by the `max_length` argument or\n    the maximum length accepted by the model if no `max_length` is provided (`max_length=None`). This will\n    truncate token by token, removing a token from the longest sequence in the pair until the proper length is\n    reached.\n  - `'only_second'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `'only_first'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `False` or `'do_not_truncate'`: no truncation is applied. This is the default behavior.\n\nThe `max_length` argument controls the length of the padding and truncation. It can be an integer or `None`, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to `max_length` is deactivated.\n\nThe following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace `truncation=True` by a `STRATEGY` selected in\n`['only_first', 'only_second', 'longest_first']`, i.e. `truncation='only_second'` or `truncation='longest_first'` to control how both sequences in the pair are truncated as detailed before."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 18b8e036-a6e7-4d49-8d33-636687d3bf73
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/pad_truncation", "title": "Padding and truncation", "tokens": 1364, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"18b8e036-a6e7-4d49-8d33-636687d3bf73\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"c3429a14-958d-4f31-8c2b-a32ff4a1a597\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"fcbf39d53d83c6f064cccf34a7654672d76b41edb3c0d340b128169d06afb319\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b574880f-18f4-47cd-b3ad-45061fb60e8a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c63f8575ca6beac956d1366d237eb2a23446ea0293863d38ffdcf7640921b0a2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1613, \"end_char_idx\": 6842, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "doc_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "ref_doc_id": "c3429a14-958d-4f31-8c2b-a32ff4a1a597", "chroma:document": "This will\n    truncate token by token, removing a token from the longest sequence in the pair until the proper length is\n    reached.\n  - `'only_second'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `'only_first'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `False` or `'do_not_truncate'`: no truncation is applied. This is the default behavior.\n\nThe `max_length` argument controls the length of the padding and truncation. It can be an integer or `None`, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to `max_length` is deactivated.\n\nThe following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace `truncation=True` by a `STRATEGY` selected in\n`['only_first', 'only_second', 'longest_first']`, i.e. `truncation='only_second'` or `truncation='longest_first'` to control how both sequences in the pair are truncated as detailed before.\n\n| Truncation                           | Padding                           | Instruction                                                                                 |\n|--------------------------------------|-----------------------------------|---------------------------------------------------------------------------------------------|\n| no truncation                        | no padding                        | `tokenizer(batch_sentences)`                                                           |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True)` or                                          |\n|                                      |                                   | `tokenizer(batch_sentences, padding='longest')`                                        |\n|                                      | padding to max model input length | `tokenizer(batch_sentences, padding='max_length')`                                     |\n|                                      | padding to specific length        | `tokenizer(batch_sentences, padding='max_length', max_length=42)`                      |\n|                                      | padding to a multiple of a value  | `tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)`                        |\n| truncation to max model input length | no padding                        | `tokenizer(batch_sentences, truncation=True)` or                                       |\n|                                      |                                   | `tokenizer(batch_sentences, truncation=STRATEGY)`                                      |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True, truncation=True)` or                         |\n|                                      |                                   | `tokenizer(batch_sentences, padding=True, truncation=STRATEGY)`                        |\n|                                      | padding to max model input length | `tokenizer(batch_sentences, padding='max_length', truncation=True)` or                 |\n|                                      |                                   | `tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)`                |\n|                                      | padding to specific length        | Not possible                                                                                |\n| truncation to specific length        | no padding                        | `tokenizer(batch_sentences, truncation=True, max_length=42)` or                        |\n|                                      |                                   | `tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)`                       |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)` or          |\n|                                      |                                   | `tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)`         |\n|                                      | padding to max model input length | Not possible                                                                                |\n|                                      | padding to specific length        | `tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)` or  |\n|                                      |                                   | `tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)` |"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 54b25fbf-44ae-44d9-9fce-71f09c126697
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"54b25fbf-44ae-44d9-9fce-71f09c126697\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"122ef393-e351-4677-b651-6d54dbe3a0b1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"534c67e5180c6deb34ef09f181d5c4b2342c6858616dcd8c6733755a0d246a9e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3948, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "# Best Practices for Generation with Cache\n\nEfficient caching is crucial for optimizing the performance of models in various generative tasks,\nincluding text generation, translation, summarization and other transformer-based applications.\nEffective caching helps reduce computation time and improve response rates, especially in real-time or resource-intensive applications.\n\nTransformers support various caching methods, leveraging \"Cache\" classes to abstract and manage the caching logic.\nThis document outlines best practices for using these classes to maximize performance and efficiency.\nCheck out all the available `Cache` classes in the [API documentation](./internal/generation_utils.md).\n\n## What is Cache and why we should care?\n\nImagine you\u2019re having a conversation with someone, and instead of remembering what was said previously, you have to start from scratch every time you respond. This would be slow and inefficient, right? In the world of Transformer models, a similar concept applies, and that's where Caching keys and values come into play. From now on, I'll refer to the concept as KV Cache.\n\nKV cache is needed to optimize the generation in autoregressive models, where the model predicts text token by token. This process can be slow since the model can generate only one token at a time, and each new prediction is dependent on the previous context. That means, to predict token number 1000 in the generation, you need information from the previous 999 tokens, which comes in the form of some matrix multiplications across the representations of those tokens. But to predict token number 1001, you also need the same information from the first 999 tokens, plus additional information from token number 1000. That is where key-value cache is used to optimize the sequential generation process by storing previous calculations to reuse in subsequent tokens, so they don't need to be computed again.\n\nMore concretely, key-value cache acts as a memory bank for these generative models, where the model stores key-value pairs derived from self-attention layers for previously processed tokens. By storing this information, the model can avoid redundant computations and instead retrieve keys and values of previous tokens from the cache.\n\n<details>\n  <summary><em>For the Curious Minds Who Like to Dive Deep</em></summary>\n\n  ### Under the Hood: How Cache Object Works in Attention Mechanism\n\n  When utilizing a cache object in the input, the Attention module performs several critical steps to integrate past and present information seamlessly.\n\n  The Attention module concatenates the current key-values with the past key-values stored in the cache. This results in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`. Essentially, the past and current key-values are combined to compute attention scores, ensuring that the model considers both previous context and new input. The concatenated key-values are used to compute the attention scores resulting in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`.\n\n  Therefore, when iteratively calling `forward()` instead of the `generate()` method, it\u2019s crucial to ensure that the attention mask shape matches the combined length of past and current key-values. The attention mask should have the shape `(batch_size, past_kv_length + new_tokens_length)`. This is usually handled internally when you call `generate()` method. If you want to implement your own generation loop with Cache classes, take this into consideration and prepare the attention mask to hold values to current and past tokens.\n\n  <Tip warning={true}>\n\n  One important concept you need to know when writing your own generation loop, is `cache_position`. In case you want to reuse an already filled Cache object by calling `forward()`, you have to pass in a valid `cache_position` which will indicate the positions of inputs in the sequence."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 122ef393-e351-4677-b651-6d54dbe3a0b1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"122ef393-e351-4677-b651-6d54dbe3a0b1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"54b25fbf-44ae-44d9-9fce-71f09c126697\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"8be2f00d864f2e3330be7247b09d8c8cb4c44d7aafd6a099644ac11f4bd13d39\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9002990c-3cab-464e-a9dc-924573a9e94d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6b9e19fd4fec58451ecd7f8b9e3040cdca9b71c6b3ef314385d85e0728a5145e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2114, \"end_char_idx\": 4227, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "By storing this information, the model can avoid redundant computations and instead retrieve keys and values of previous tokens from the cache.\n\n<details>\n  <summary><em>For the Curious Minds Who Like to Dive Deep</em></summary>\n\n  ### Under the Hood: How Cache Object Works in Attention Mechanism\n\n  When utilizing a cache object in the input, the Attention module performs several critical steps to integrate past and present information seamlessly.\n\n  The Attention module concatenates the current key-values with the past key-values stored in the cache. This results in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`. Essentially, the past and current key-values are combined to compute attention scores, ensuring that the model considers both previous context and new input. The concatenated key-values are used to compute the attention scores resulting in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`.\n\n  Therefore, when iteratively calling `forward()` instead of the `generate()` method, it\u2019s crucial to ensure that the attention mask shape matches the combined length of past and current key-values. The attention mask should have the shape `(batch_size, past_kv_length + new_tokens_length)`. This is usually handled internally when you call `generate()` method. If you want to implement your own generation loop with Cache classes, take this into consideration and prepare the attention mask to hold values to current and past tokens.\n\n  <Tip warning={true}>\n\n  One important concept you need to know when writing your own generation loop, is `cache_position`. In case you want to reuse an already filled Cache object by calling `forward()`, you have to pass in a valid `cache_position` which will indicate the positions of inputs in the sequence. Note that `cache_position` is not affected by padding, and always adds one more position for each token. For example, if key/value cache contains 10 tokens (no matter how many of it is a pad token), the cache position for the next token should be `torch.tensor([10])`.\n\n  </Tip>"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 9002990c-3cab-464e-a9dc-924573a9e94d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9002990c-3cab-464e-a9dc-924573a9e94d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"122ef393-e351-4677-b651-6d54dbe3a0b1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c5a4416e41be148d32c415dcf40646220a64c0a403f5784c8637d19250f26a77\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52b82913-23c4-4593-b245-fea22360a132\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7d4d5ab9460b0d5100cb8d590f166b3ee092fd7da15d194839a7a6410a574ed5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2569, \"end_char_idx\": 6117, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "The Attention module concatenates the current key-values with the past key-values stored in the cache. This results in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`. Essentially, the past and current key-values are combined to compute attention scores, ensuring that the model considers both previous context and new input. The concatenated key-values are used to compute the attention scores resulting in attention weights of shape `(new_tokens_length, past_kv_length + new_tokens_length)`.\n\n  Therefore, when iteratively calling `forward()` instead of the `generate()` method, it\u2019s crucial to ensure that the attention mask shape matches the combined length of past and current key-values. The attention mask should have the shape `(batch_size, past_kv_length + new_tokens_length)`. This is usually handled internally when you call `generate()` method. If you want to implement your own generation loop with Cache classes, take this into consideration and prepare the attention mask to hold values to current and past tokens.\n\n  <Tip warning={true}>\n\n  One important concept you need to know when writing your own generation loop, is `cache_position`. In case you want to reuse an already filled Cache object by calling `forward()`, you have to pass in a valid `cache_position` which will indicate the positions of inputs in the sequence. Note that `cache_position` is not affected by padding, and always adds one more position for each token. For example, if key/value cache contains 10 tokens (no matter how many of it is a pad token), the cache position for the next token should be `torch.tensor([10])`.\n\n  </Tip>\n\n\n  See an example below for how to implement your own generation loop.\n    \n  ```python\n  >>> import torch\n  >>> from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache\n \n  >>> model_id = \"meta-llama/Llama-2-7b-chat-hf\"\n  >>> model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=\"cuda:0\")\n  >>> tokenizer = AutoTokenizer.from_pretrained(model_id)\n\n  >>> past_key_values = DynamicCache()\n  >>> messages = [{\"role\": \"user\", \"content\": \"Hello, what's your name.\"}]\n  >>> inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors=\"pt\", return_dict=True).to(\"cuda:0\")\n\n  >>> generated_ids = inputs.input_ids\n  >>> cache_position = torch.arange(inputs.input_ids.shape[1], dtype=torch.int64, device=\"cuda:0\")\n  >>> max_new_tokens = 10\n\n  >>> for _ in range(max_new_tokens):\n  ...     outputs = model(**inputs, cache_position=cache_position, past_key_values=past_key_values, use_cache=True)     \n  ...     # Greedily sample one next token\n  ...     next_token_ids = outputs.logits[:, -1:].argmax(-1)\n  ...     generated_ids = torch.cat([generated_ids, next_token_ids], dim=-1)   \n  ...\n  ...     # Prepare inputs for the next generation step by leaaving unprocessed tokens, in our case we have only one new token\n  ...     # and expanding attn mask for the new token, as explained above\n  ...     attention_mask = inputs[\"attention_mask\"]\n  ...     attention_mask = torch.cat([attention_mask, attention_mask.new_ones((attention_mask.shape[0], 1))], dim=-1)\n  ...     inputs = {\"input_ids\": next_token_ids, \"attention_mask\": attention_mask}\n  ...     cache_position = cache_position[-1:] + 1 # add one more position for the next token\n\n  >>> print(tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0])\n  \"[INST] Hello, what's your name. [/INST]  Hello! My name is LLaMA,\"\n  ```\n\n</details>"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 52b82913-23c4-4593-b245-fea22360a132
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"52b82913-23c4-4593-b245-fea22360a132\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9002990c-3cab-464e-a9dc-924573a9e94d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9e3b991d597d22c2e0e1b54202cc3f63961d1a49d916f721f9f8f5094f120edd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"38430138-73d7-4305-821b-b959870ba674\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"66ddfb932d517cf286fda87ab14f95d1732cf5f2efa4eb809e5c2844b1bb10fd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6121, \"end_char_idx\": 9664, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "## Generate with Cache\n\nIn \ud83e\udd17 Transformers, we support various Cache types to optimize the performance across different models and tasks. By default, all models generate with caching,\nwith the [`~DynamicCache`] class being the default cache for most models. It allows us to dynamically grow cache size, by saving more and more keys and values as we generate. If for some reason you don't want to use caches, you can pass `use_cache=False` into the `generate()` method.\n\nRefer to the table below to see the difference between cache types and choose the one that suits best for your use-case.\n\n| Cache Type          | Memory Efficient | Supports torch.compile() | Initialization Recommended | Latency  |  Long Context Generation |\n|---------------------|------------------|--------------------------|----------------------------|----------|--------------------------|\n| Dynamic Cache       |      No          |        No                |         No                 |   Mid    |     No                   |\n| Static Cache        |      No          |        Yes               |         Yes                |   High   |     No                   |\n| Quantized Cache     |      Yes         |        No                |         No                 |   Low    |     Yes                  |\n| Offloaded Cache     |      Yes         |        No                |         No                 |   Low    |     No                   |\n| Sliding Window Cache|      No          |        Yes               |         Yes                |   High   |     No                   |\n| Sink Cache          |      Yes         |        No                |         Yes                |   Mid    |     Yes                  |\n\n\nThese cache classes can be set with a `cache_implementation` argument when generating. To learn about the available options for the cache_implementation flag, please refer to the [API Documentation](./main_classes/text_generation.md#transformers.GenerationConfig). Now, let's explore each cache type in detail and see how to use them. Note that the below examples are for decoder-only Tranformer-based models. We also support [\"Model-Specific Cache\"] classes for models such as Mamba or Jamba, keep reading for more details.\n\n### Quantized Cache\n\nThe key and value cache can occupy a large portion of memory, becoming a [bottleneck for long-context generation](https://huggingface.co/blog/llama31#inference-memory-requirements), especially for Large Language Models.\nQuantizing the cache when using `generate()` can significantly reduce memory requirements at the cost of speed.\n\nKV Cache quantization in `transformers` is largely inspired by the paper [\"KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache\"](https://arxiv.org/abs/2402.02750) and currently supports [`~QuantoQuantizedCache`] and [`~HQQQuantizedCache`] classes. For more information on the inner workings see the paper.\n\nTo enable quantization of the key-value cache, one needs to indicate `cache_implementation=\"quantized\"` in the `generation_config`.\nQuantization related arguments should be passed to the `generation_config` either as a `dict` or an instance of a [`~QuantizedCacheConfig`] class.\nOne has to indicate which quantization backend to use in the [`~QuantizedCacheConfig`], the default is `quanto`.\n\n<Tip warning={true}>\n\nCache quantization can be detrimental in terms of latency if the context length is short and there is enough GPU VRAM available to run without cache quantization. It is recommended to seek balance between memory efficiency and latency.\n</Tip>"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 38430138-73d7-4305-821b-b959870ba674
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"38430138-73d7-4305-821b-b959870ba674\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"52b82913-23c4-4593-b245-fea22360a132\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eacbba2668f333eb8e7c8ce5878486c9ff4d25a9832bc51b4aed2676a69f2270\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6a276a67-1308-4779-976d-107290ac366b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"66321d2c8b0a79041d2bd1be2c5ad781b7f8cb438580d2ea2b1b3851e9520af1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9667, \"end_char_idx\": 12713, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\", torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"I like rock music because\", return_tensors=\"pt\").to(model.device)\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=20, cache_implementation=\"quantized\", cache_config={\"nbits\": 4, \"backend\": \"quanto\"})\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nI like rock music because it's loud and energetic. It's a great way to express myself and rel\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=20)\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nI like rock music because it's loud and energetic. I like to listen to it when I'm feeling\n```\n\n## OffloadedCache\n\nSimilarly to KV cache quantization, [`~OffloadedCache`] strategy aims to reduce GPU VRAM usage.\nIt does so by moving the KV cache for most layers to the CPU.\nAs the model's `forward()` method iterates over the layers, this strategy maintains the current layer cache on the GPU.\nAt the same time it asynchronously prefetches the next layer cache as well as sending the previous layer cache back to the CPU.\nUnlike KV cache quantization, this strategy always produces the same result as the default KV cache implementation.\nThus, it can serve as a drop-in replacement or a fallback for it.\n\nDepending on your model and the characteristics of your generation task (size of context, number of generated tokens, number of beams, etc.)\nyou may notice a small degradation in generation throughput compared to the default KV cache implementation.\n\nTo enable KV cache offloading, pass `cache_implementation=\"offloaded\"` in the `generation_config` or directky to the `generate()` call.\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n>>> ckpt = \"microsoft/Phi-3-mini-4k-instruct\"\n\n>>> tokenizer = AutoTokenizer.from_pretrained(ckpt)\n>>> model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"Fun fact: The shortest\", return_tensors=\"pt\").to(model.device)\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=23, cache_implementation=\"offloaded\")\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nFun fact: The shortest war in history was between Britain and Zanzibar on August 27, 1896.\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=23)\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nFun fact: The shortest war in history was between Britain and Zanzibar on August 27, 1896.\n```\n\n<Tip warning={true}>\n\nCache offloading requires a GPU and can be slower than dynamic KV cache. Use it if you are getting CUDA out of memory errors.\n\n</Tip>\n\nThe example below shows how KV cache offloading can be used as a fallback strategy."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 6a276a67-1308-4779-976d-107290ac366b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"6a276a67-1308-4779-976d-107290ac366b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"38430138-73d7-4305-821b-b959870ba674\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"764c5fd1ed1a8c4a1169f414ee46d024558e85dcbd66adb6c067926527935107\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"68e0bb14-2e04-4ff3-ae6a-ee333b620f48\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9926587d4d0cba90492ee1c448c1c0ed8bbd6f6b0502c1bc59b309d27a048b61\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11028, \"end_char_idx\": 14083, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "Unlike KV cache quantization, this strategy always produces the same result as the default KV cache implementation.\nThus, it can serve as a drop-in replacement or a fallback for it.\n\nDepending on your model and the characteristics of your generation task (size of context, number of generated tokens, number of beams, etc.)\nyou may notice a small degradation in generation throughput compared to the default KV cache implementation.\n\nTo enable KV cache offloading, pass `cache_implementation=\"offloaded\"` in the `generation_config` or directky to the `generate()` call.\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n>>> ckpt = \"microsoft/Phi-3-mini-4k-instruct\"\n\n>>> tokenizer = AutoTokenizer.from_pretrained(ckpt)\n>>> model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"Fun fact: The shortest\", return_tensors=\"pt\").to(model.device)\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=23, cache_implementation=\"offloaded\")\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nFun fact: The shortest war in history was between Britain and Zanzibar on August 27, 1896.\n\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=23)\n>>> print(tokenizer.batch_decode(out, skip_special_tokens=True)[0])\nFun fact: The shortest war in history was between Britain and Zanzibar on August 27, 1896.\n```\n\n<Tip warning={true}>\n\nCache offloading requires a GPU and can be slower than dynamic KV cache. Use it if you are getting CUDA out of memory errors.\n\n</Tip>\n\nThe example below shows how KV cache offloading can be used as a fallback strategy.\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n>>> def resilient_generate(model, *args, **kwargs):\n...     oom = False\n...     try:\n...         return model.generate(*args, **kwargs)\n...     except torch.cuda.OutOfMemoryError as e:\n...         print(e)\n...         print(\"retrying with cache_implementation='offloaded'\")\n...         oom = True\n...     if oom:\n...         torch.cuda.empty_cache()\n...         kwargs[\"cache_implementation\"] = \"offloaded\"\n...         return model.generate(*args, **kwargs)\n...\n...\n>>> ckpt = \"microsoft/Phi-3-mini-4k-instruct\"\n>>> tokenizer = AutoTokenizer.from_pretrained(ckpt)\n>>> model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(\"cuda:0\")\n>>> prompt = [\"okay \"*1000 + \"Fun fact: The most\"]\n>>> inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n>>> beams = { \"num_beams\": 40, \"num_beam_groups\": 40, \"num_return_sequences\": 40, \"diversity_penalty\": 1.0, \"max_new_tokens\": 23, \"early_stopping\": True, }\n>>> out = resilient_generate(model, **inputs, **beams)\n>>> responses = tokenizer.batch_decode(out[:,-28:], skip_special_tokens=True)\n```\n\nOn a GPU with 50 GB of RAM, running this code will print\n```\nCUDA out of memory. Tried to allocate 4.83 GiB. GPU\nretrying with cache_implementation='offloaded'\n```\nbefore successfully generating 40 beams."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 68e0bb14-2e04-4ff3-ae6a-ee333b620f48
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"68e0bb14-2e04-4ff3-ae6a-ee333b620f48\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6a276a67-1308-4779-976d-107290ac366b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ea389fa8e56325829da630efcb9d2d58d5e0559805335734d39029d9550cef30\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dbf9b9b2-cbfd-4d2b-905f-2f8dc8580ef5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d237030fe8bed58da32504f51953913acaed10a66085cddd7c90cda8871b66ce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12568, \"end_char_idx\": 15764, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "Use it if you are getting CUDA out of memory errors.\n\n</Tip>\n\nThe example below shows how KV cache offloading can be used as a fallback strategy.\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n>>> def resilient_generate(model, *args, **kwargs):\n...     oom = False\n...     try:\n...         return model.generate(*args, **kwargs)\n...     except torch.cuda.OutOfMemoryError as e:\n...         print(e)\n...         print(\"retrying with cache_implementation='offloaded'\")\n...         oom = True\n...     if oom:\n...         torch.cuda.empty_cache()\n...         kwargs[\"cache_implementation\"] = \"offloaded\"\n...         return model.generate(*args, **kwargs)\n...\n...\n>>> ckpt = \"microsoft/Phi-3-mini-4k-instruct\"\n>>> tokenizer = AutoTokenizer.from_pretrained(ckpt)\n>>> model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(\"cuda:0\")\n>>> prompt = [\"okay \"*1000 + \"Fun fact: The most\"]\n>>> inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n>>> beams = { \"num_beams\": 40, \"num_beam_groups\": 40, \"num_return_sequences\": 40, \"diversity_penalty\": 1.0, \"max_new_tokens\": 23, \"early_stopping\": True, }\n>>> out = resilient_generate(model, **inputs, **beams)\n>>> responses = tokenizer.batch_decode(out[:,-28:], skip_special_tokens=True)\n```\n\nOn a GPU with 50 GB of RAM, running this code will print\n```\nCUDA out of memory. Tried to allocate 4.83 GiB. GPU\nretrying with cache_implementation='offloaded'\n```\nbefore successfully generating 40 beams.\n\n\n\n### Static Cache\n\nSince the \"DynamicCache\" dynamically grows with each generation step, it prevents you from taking advantage of JIT optimizations. The [`~StaticCache`] pre-allocates \na specific maximum size for the keys and values, allowing you to generate up to the maximum length without having to modify cache size. Check the below usage example.\n\nFor more examples with Static Cache and JIT compilation, take a look at [StaticCache & torchcompile](./llm_optims.md#static-kv-cache-and-torchcompile)\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\", torch_dtype=torch.float16, device_map=\"auto\")\n>>> inputs = tokenizer(\"Hello, my name is\", return_tensors=\"pt\").to(model.device)\n\n>>> # simply pass the cache implementation=\"static\"\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=20, cache_implementation=\"static\")\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"Hello, my name is [Your Name], and I am a [Your Profession] with [Number of Years] of\"\n```\n\n### Sliding Window Cache\n\nAs the name suggests, this cache type implements a sliding window over previous keys and values, retaining only the last `sliding_window` tokens. It should be used with models like Mistral that support sliding window attention. Additionally, similar to Static Cache, this one is JIT-friendly and can be used with the same compile tecniques as Static Cache.\n\nNote that you can use this cache only for models that support sliding window, e.g. Mistral models."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: dbf9b9b2-cbfd-4d2b-905f-2f8dc8580ef5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"dbf9b9b2-cbfd-4d2b-905f-2f8dc8580ef5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"68e0bb14-2e04-4ff3-ae6a-ee333b620f48\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9b6fd0b92487c02b6c5fc4b98373620d8aba00716b12ef9f79ca546d45174a4b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4e694473-b3f3-4b8f-97ba-e19277942fb1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1f5a0e6d56913157177267eee6c9259107bb3667020b612074c13fb2ccea0729\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14087, \"end_char_idx\": 17422, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "### Static Cache\n\nSince the \"DynamicCache\" dynamically grows with each generation step, it prevents you from taking advantage of JIT optimizations. The [`~StaticCache`] pre-allocates \na specific maximum size for the keys and values, allowing you to generate up to the maximum length without having to modify cache size. Check the below usage example.\n\nFor more examples with Static Cache and JIT compilation, take a look at [StaticCache & torchcompile](./llm_optims.md#static-kv-cache-and-torchcompile)\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\", torch_dtype=torch.float16, device_map=\"auto\")\n>>> inputs = tokenizer(\"Hello, my name is\", return_tensors=\"pt\").to(model.device)\n\n>>> # simply pass the cache implementation=\"static\"\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=20, cache_implementation=\"static\")\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"Hello, my name is [Your Name], and I am a [Your Profession] with [Number of Years] of\"\n```\n\n### Sliding Window Cache\n\nAs the name suggests, this cache type implements a sliding window over previous keys and values, retaining only the last `sliding_window` tokens. It should be used with models like Mistral that support sliding window attention. Additionally, similar to Static Cache, this one is JIT-friendly and can be used with the same compile tecniques as Static Cache.\n\nNote that you can use this cache only for models that support sliding window, e.g. Mistral models. \n\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\", torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"Yesterday I was on a rock concert and.\", return_tensors=\"pt\").to(model.device)\n\n>>> # can be used by passing in cache implementation\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=30, cache_implementation=\"sliding_window\")\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"Yesterday I was on a rock concert and. I was so excited to see my favorite band. I was so excited that I was jumping up and down and screaming. I was so excited that I\"\n```\n\n### Sink Cache\n\nSink Cache was introduced in [\"Efficient Streaming Language Models with Attention Sinks\"](https://arxiv.org/abs/2309.17453). It allows you to generate long sequences of text (\"infinite length\" according to the paper) without any fine-tuning. That is achieved by smart handling of previous keys and values, specifically it retains a few initial tokens from the sequence, called \"sink tokens\". This is based on the observation that these initial tokens attract a significant portion of attention scores during the generation process. Tokens that come after \"sink tokens\" are discarded on a sliding windowed basis, keeping only the latest `window_size` tokens. By keeping these initial tokens as \"attention sinks,\" the model maintains stable performance even when dealing with very long texts, thus discarding most of the previous knowledge."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 4e694473-b3f3-4b8f-97ba-e19277942fb1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4e694473-b3f3-4b8f-97ba-e19277942fb1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dbf9b9b2-cbfd-4d2b-905f-2f8dc8580ef5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e925d6a9c35d872d4d6dd70ceaefe19132aa6e3642ec809098b0d1a8482d9baf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d47dcb1e-4a53-491d-ba9a-b7f05b45c835\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a4f1a3e3617a329db5cd00660825ab2092fbaef121f1282564ed73edfb82394d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15768, \"end_char_idx\": 19046, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"mistralai/Mistral-7B-v0.1\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"mistralai/Mistral-7B-v0.1\", torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"Yesterday I was on a rock concert and.\", return_tensors=\"pt\").to(model.device)\n\n>>> # can be used by passing in cache implementation\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=30, cache_implementation=\"sliding_window\")\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"Yesterday I was on a rock concert and. I was so excited to see my favorite band. I was so excited that I was jumping up and down and screaming. I was so excited that I\"\n```\n\n### Sink Cache\n\nSink Cache was introduced in [\"Efficient Streaming Language Models with Attention Sinks\"](https://arxiv.org/abs/2309.17453). It allows you to generate long sequences of text (\"infinite length\" according to the paper) without any fine-tuning. That is achieved by smart handling of previous keys and values, specifically it retains a few initial tokens from the sequence, called \"sink tokens\". This is based on the observation that these initial tokens attract a significant portion of attention scores during the generation process. Tokens that come after \"sink tokens\" are discarded on a sliding windowed basis, keeping only the latest `window_size` tokens. By keeping these initial tokens as \"attention sinks,\" the model maintains stable performance even when dealing with very long texts, thus discarding most of the previous knowledge.\n\nUnlike other cache classes, this one can't be used directly by indicating a `cache_implementation`. You have to initialize the Cache before calling on `generate()` as follows.\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\", torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"This is a long story about unicorns, fairies and magic.\", return_tensors=\"pt\").to(model.device)\n\n>>> # get our cache, specify number of sink tokens and window size\n>>> # Note that window size already includes sink tokens, so has to be larger\n>>> past_key_values = SinkCache(window_length=256, num_sink_tokens=4)\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=30, past_key_values=past_key_values)\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"This is a long story about unicorns, fairies and magic. It is a fantasy world where unicorns and fairies live together in harmony. The story follows a young girl named Lily\"\n```\n\n### Encoder-Decoder Cache\n\nThe [`~EncoderDecoderCache`] is a wrapper designed to handle the caching needs of encoder-decoder models. This cache type is specifically built to manage both self-attention and cross-attention caches, ensuring storage and retrieval of past key/values required for these complex models. Cool thing about Encoder-Decoder Cache is that you can set different cache types for the encoder and for the decoder, depending on your use case."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: d47dcb1e-4a53-491d-ba9a-b7f05b45c835
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d47dcb1e-4a53-491d-ba9a-b7f05b45c835\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4e694473-b3f3-4b8f-97ba-e19277942fb1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"4b5f523380bfdb361e7c37869d7986ae455c872a99d225cf48e6d476fa0d7e52\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"51c91c6b-a86b-45c2-95b4-c6e97460072d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c0b4afc16781aede16d569f054471650d8f67084c69ad536b8687852a3a016cf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17424, \"end_char_idx\": 20905, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "Unlike other cache classes, this one can't be used directly by indicating a `cache_implementation`. You have to initialize the Cache before calling on `generate()` as follows.\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer, AutoModelForCausalLM, SinkCache\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\")\n>>> model = AutoModelForCausalLM.from_pretrained(\"meta-llama/Llama-2-7b-chat-hf\", torch_dtype=torch.float16).to(\"cuda:0\")\n>>> inputs = tokenizer(\"This is a long story about unicorns, fairies and magic.\", return_tensors=\"pt\").to(model.device)\n\n>>> # get our cache, specify number of sink tokens and window size\n>>> # Note that window size already includes sink tokens, so has to be larger\n>>> past_key_values = SinkCache(window_length=256, num_sink_tokens=4)\n>>> out = model.generate(**inputs, do_sample=False, max_new_tokens=30, past_key_values=past_key_values)\n>>> tokenizer.batch_decode(out, skip_special_tokens=True)[0]\n\"This is a long story about unicorns, fairies and magic. It is a fantasy world where unicorns and fairies live together in harmony. The story follows a young girl named Lily\"\n```\n\n### Encoder-Decoder Cache\n\nThe [`~EncoderDecoderCache`] is a wrapper designed to handle the caching needs of encoder-decoder models. This cache type is specifically built to manage both self-attention and cross-attention caches, ensuring storage and retrieval of past key/values required for these complex models. Cool thing about Encoder-Decoder Cache is that you can set different cache types for the encoder and for the decoder, depending on your use case. Currently this cache is only supported in [Whisper](./model_doc/whisper.md) models but we will be adding more models soon. \n\nIn terms of usage, there is nothing special to be done and calling `generate()` or `forward()` will handle everything for you.\n\n\n### Model-specific Cache Classes\n\nSome models require storing previous keys, values, or states in a specific way, and the above cache classes cannot be used. For such cases, we have several specialized cache classes that are designed for specific models. These models only accept their own dedicated cache classes and do not support using any other cache types. Some examples include [`~HybridCache`] for [Gemma2](./model_doc/gemma2.md) series models or [`~MambaCache`] for [Mamba](./model_doc/mamba.md) architecture models.\n\n\n## Iterative Generation with Cache\n\nWe have seen how to use each of the cache types when generating. What if you want to use cache in iterative generation setting, for example in applications like chatbots, where interactions involve multiple turns and continuous back-and-forth exchanges. Iterative generation with cache allows these systems to handle ongoing conversations effectively without reprocessing the entire context at each step. But there are some tips that you should know before you start implementing:\n\nThe general format when doing iterative generation is as below. First you have to initialize an empty cache of the type you want, and you can start feeding in new prompts iteratively. Keeping track of dialogues history and formatting can be done with chat templates, read more on that in [chat_templating](./chat_templating.md)\n\nIn case you are using Sink Cache, you have to crop your inputs to that maximum length because Sink Cache can generate text longer than its maximum window size, but it expects the first input to not exceed the maximum cache length."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 51c91c6b-a86b-45c2-95b4-c6e97460072d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"51c91c6b-a86b-45c2-95b4-c6e97460072d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d47dcb1e-4a53-491d-ba9a-b7f05b45c835\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0b2e32cdf24340c109c38f21d3bda3ee750405eefc8da9d39a5e2e4fe5b86eb7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4760620d-db8d-4bf9-9dfb-fa1485589418\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dfab3a75a4bee2faa5fa2eb24d82db8141a4d1ddc1ceb2d90878bce52f2a5843\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19047, \"end_char_idx\": 22739, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "Currently this cache is only supported in [Whisper](./model_doc/whisper.md) models but we will be adding more models soon. \n\nIn terms of usage, there is nothing special to be done and calling `generate()` or `forward()` will handle everything for you.\n\n\n### Model-specific Cache Classes\n\nSome models require storing previous keys, values, or states in a specific way, and the above cache classes cannot be used. For such cases, we have several specialized cache classes that are designed for specific models. These models only accept their own dedicated cache classes and do not support using any other cache types. Some examples include [`~HybridCache`] for [Gemma2](./model_doc/gemma2.md) series models or [`~MambaCache`] for [Mamba](./model_doc/mamba.md) architecture models.\n\n\n## Iterative Generation with Cache\n\nWe have seen how to use each of the cache types when generating. What if you want to use cache in iterative generation setting, for example in applications like chatbots, where interactions involve multiple turns and continuous back-and-forth exchanges. Iterative generation with cache allows these systems to handle ongoing conversations effectively without reprocessing the entire context at each step. But there are some tips that you should know before you start implementing:\n\nThe general format when doing iterative generation is as below. First you have to initialize an empty cache of the type you want, and you can start feeding in new prompts iteratively. Keeping track of dialogues history and formatting can be done with chat templates, read more on that in [chat_templating](./chat_templating.md)\n\nIn case you are using Sink Cache, you have to crop your inputs to that maximum length because Sink Cache can generate text longer than its maximum window size, but it expects the first input to not exceed the maximum cache length.  \n\n\n```python\n>>> import torch\n>>> from transformers import AutoTokenizer,AutoModelForCausalLM\n>>> from transformers.cache_utils import (\n>>>     DynamicCache,\n>>>     SinkCache,\n>>>     StaticCache,\n>>>     SlidingWindowCache,\n>>>     QuantoQuantizedCache,\n>>>     QuantizedCacheConfig,\n>>> )\n\n>>> model_id = \"meta-llama/Llama-2-7b-chat-hf\"\n>>> model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map='auto')\n>>> tokenizer = AutoTokenizer.from_pretrained(model_id)\n\n>>> user_prompts = [\"Hello, what's your name?\", \"Btw, yesterday I was on a rock concert.\"]\n\n>>> past_key_values = DynamicCache()\n>>> max_cache_length = past_key_values.get_max_length()\n\n>>> messages = []\n>>> for prompt in user_prompts:\n...     messages.append({\"role\": \"user\", \"content\": prompt})\n...     inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors=\"pt\", return_dict=True).to(model.device)\n...     if isinstance(past_key_values, SinkCache):\n...         inputs = {k: v[:, -max_cache_length:] for k, v in inputs.items()}\n... \n...     input_length = inputs[\"input_ids\"].shape[1]\n...     \n...     outputs = model.generate(**inputs, do_sample=False, max_new_tokens=256, past_key_values=past_key_values)\n...     completion = tokenizer.decode(outputs[0, input_length: ], skip_special_tokens=True)\n...     messages.append({\"role\": \"assistant\", \"content\": completion})\n\nprint(messages)\n[{'role': 'user', 'content': \"Hello, what's your name?\"}, {'role': 'assistant', 'content': \" Hello! My name is LLaMA, I'm a large language model trained by a team of researcher at Meta AI. \ud83d\ude0a\"}, {'role': 'user', 'content': 'Btw, yesterday I was on a rock concert.'}, {'role': 'assistant', 'content': ' Oh, cool! That sounds like a lot of fun! \ud83c\udf89 Did you enjoy the concert? What was the band like? \ud83e\udd14'}]\n```"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 4760620d-db8d-4bf9-9dfb-fa1485589418
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/kv_cache", "title": "Best Practices for Generation with Cache", "tokens": 5292, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4760620d-db8d-4bf9-9dfb-fa1485589418\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9ad763d2-1ae8-4e91-a34f-c05972c1a583\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e68b35583673d8d22958554efdb9a7a1dccc9600935a4975537e48d1bda7b56c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"51c91c6b-a86b-45c2-95b4-c6e97460072d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/kv_cache\", \"title\": \"Best Practices for Generation with Cache\", \"tokens\": 5292, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"20dccc6f086c7c9461449f3996ddc227a0e41b487810ae4ed0012f90e64e8eae\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22742, \"end_char_idx\": 23042, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "ref_doc_id": "9ad763d2-1ae8-4e91-a34f-c05972c1a583", "chroma:document": "## Re-use Cache to continue generation\n\nSometimes you would want to fist fill-in cache object with key/values for certain prefix prompt and re-use it several times to generate different sequences from it. We are working hard on adding this feature to \ud83e\udd17 Transformers and will update this section soon."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: ab10489c-9537-436c-b266-80b2efe7b587
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/accelerate", "title": "Distributed training with \ud83e\udd17 Accelerate", "tokens": 1054, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"ab10489c-9537-436c-b266-80b2efe7b587\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3aa99498-7863-4cf7-bc51-e5f465bc7f4a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"839d0123fd4e6631f69648ba6ca5163ea5abc3628df918cbee14aaa6ef50dfd4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cdc660ae-1551-48cd-a7b9-a339b935dbe4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d72d682e22186a9be416d73902d80574c8e1bb97898d735edede859bfb0be1ae\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3394, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "doc_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "ref_doc_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "chroma:document": "# Distributed training with \ud83e\udd17 Accelerate\n\nAs models get bigger, parallelism has emerged as a strategy for training larger models on limited hardware and accelerating training speed by several orders of magnitude. At Hugging Face, we created the [\ud83e\udd17 Accelerate](https://huggingface.co/docs/accelerate) library to help users easily train a \ud83e\udd17 Transformers model on any type of distributed setup, whether it is multiple GPU's on one machine or multiple GPU's across several machines. In this tutorial, learn how to customize your native PyTorch training loop to enable training in a distributed environment.\n\n## Setup\n\nGet started by installing \ud83e\udd17 Accelerate:\n\n```bash\npip install accelerate\n```\n\nThen import and create an [`~accelerate.Accelerator`] object. The [`~accelerate.Accelerator`] will automatically detect your type of distributed setup and initialize all the necessary components for training. You don't need to explicitly place your model on a device.\n\n```py\n>>> from accelerate import Accelerator\n\n>>> accelerator = Accelerator()\n```\n\n## Prepare to accelerate\n\nThe next step is to pass all the relevant training objects to the [`~accelerate.Accelerator.prepare`] method. This includes your training and evaluation DataLoaders, a model and an optimizer:\n\n```py\n>>> train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n...     train_dataloader, eval_dataloader, model, optimizer\n... )\n```\n\n## Backward\n\nThe last addition is to replace the typical `loss.backward()` in your training loop with \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.backward`]method:\n\n```py\n>>> for epoch in range(num_epochs):\n...     for batch in train_dataloader:\n...         outputs = model(**batch)\n...         loss = outputs.loss\n...         accelerator.backward(loss)\n\n...         optimizer.step()\n...         lr_scheduler.step()\n...         optimizer.zero_grad()\n...         progress_bar.update(1)\n```\n\nAs you can see in the following code, you only need to add four additional lines of code to your training loop to enable distributed training!\n\n```diff\n+ from accelerate import Accelerator\n  from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler\n\n+ accelerator = Accelerator()\n\n  model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n  optimizer = AdamW(model.parameters(), lr=3e-5)\n\n- device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n- model.to(device)\n\n+ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n+     train_dataloader, eval_dataloader, model, optimizer\n+ )\n\n  num_epochs = 3\n  num_training_steps = num_epochs * len(train_dataloader)\n  lr_scheduler = get_scheduler(\n      \"linear\",\n      optimizer=optimizer,\n      num_warmup_steps=0,\n      num_training_steps=num_training_steps\n  )\n\n  progress_bar = tqdm(range(num_training_steps))\n\n  model.train()\n  for epoch in range(num_epochs):\n      for batch in train_dataloader:\n-         batch = {k: v.to(device) for k, v in batch.items()}\n          outputs = model(**batch)\n          loss = outputs.loss\n-         loss.backward()\n+         accelerator.backward(loss)\n\n          optimizer.step()\n          lr_scheduler.step()\n          optimizer.zero_grad()\n          progress_bar.update(1)\n```\n\n## Train\n\nOnce you've added the relevant lines of code, launch your training in a script or a notebook like Colaboratory."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: cdc660ae-1551-48cd-a7b9-a339b935dbe4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/accelerate", "title": "Distributed training with \ud83e\udd17 Accelerate", "tokens": 1054, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"cdc660ae-1551-48cd-a7b9-a339b935dbe4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3aa99498-7863-4cf7-bc51-e5f465bc7f4a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"839d0123fd4e6631f69648ba6ca5163ea5abc3628df918cbee14aaa6ef50dfd4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ab10489c-9537-436c-b266-80b2efe7b587\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c14a49a78c6c1489bc7fb3bf51ebb898dcffbad5148dd111f4f49db77c336be0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2049, \"end_char_idx\": 4096, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "doc_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "ref_doc_id": "3aa99498-7863-4cf7-bc51-e5f465bc7f4a", "chroma:document": "```diff\n+ from accelerate import Accelerator\n  from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler\n\n+ accelerator = Accelerator()\n\n  model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n  optimizer = AdamW(model.parameters(), lr=3e-5)\n\n- device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n- model.to(device)\n\n+ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n+     train_dataloader, eval_dataloader, model, optimizer\n+ )\n\n  num_epochs = 3\n  num_training_steps = num_epochs * len(train_dataloader)\n  lr_scheduler = get_scheduler(\n      \"linear\",\n      optimizer=optimizer,\n      num_warmup_steps=0,\n      num_training_steps=num_training_steps\n  )\n\n  progress_bar = tqdm(range(num_training_steps))\n\n  model.train()\n  for epoch in range(num_epochs):\n      for batch in train_dataloader:\n-         batch = {k: v.to(device) for k, v in batch.items()}\n          outputs = model(**batch)\n          loss = outputs.loss\n-         loss.backward()\n+         accelerator.backward(loss)\n\n          optimizer.step()\n          lr_scheduler.step()\n          optimizer.zero_grad()\n          progress_bar.update(1)\n```\n\n## Train\n\nOnce you've added the relevant lines of code, launch your training in a script or a notebook like Colaboratory.\n\n### Train with a script\n\nIf you are running your training from a script, run the following command to create and save a configuration file:\n\n```bash\naccelerate config\n```\n\nThen launch your training with:\n\n```bash\naccelerate launch train.py\n```\n\n### Train with a notebook\n\n\ud83e\udd17 Accelerate can also run in a notebook if you're planning on using Colaboratory's TPUs. Wrap all the code responsible for training in a function, and pass it to [`~accelerate.notebook_launcher`]:\n\n```py\n>>> from accelerate import notebook_launcher\n\n>>> notebook_launcher(training_function)\n```\n\nFor more information about \ud83e\udd17 Accelerate and its rich features, refer to the [documentation](https://huggingface.co/docs/accelerate)."}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 19647714-3ba8-4877-b611-cd9c4bf5c093
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"19647714-3ba8-4877-b611-cd9c4bf5c093\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e176cf9a-4957-4b2a-b08a-e46ef97ba437\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5646aaa13f706a472cedfbc8fc5d1cb7e667bb5490d127cec146423143f317f6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2733, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contains specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Community\n\nThis page regroups resources around \ud83e\udd17 Transformers developed by the community.\n\n## Community resources:\n\n| Resource     |      Description      |      Author      |\n|:----------|:-------------|------:|\n| [Hugging Face Transformers Glossary Flashcards](https://www.darigovresearch.com/huggingface-transformers-glossary-flashcards) | A set of flashcards based on the [Transformers Docs Glossary](glossary) that has been put into a form which can be easily learned/revised using [Anki](https://apps.ankiweb.net/) an open source, cross platform app specifically designed for long term knowledge retention. See this [Introductory video on how to use the flashcards](https://www.youtube.com/watch?v=Dji_h7PILrw). | [Darigov Research](https://www.darigovresearch.com/) |\n\n## Community notebooks:\n\n| Notebook     |      Description      |      Author      |      |\n|:----------|:-------------|:-------------|------:|\n| [Fine-tune a pre-trained Transformer to generate lyrics](https://github.com/AlekseyKorshuk/huggingartists) | How to generate lyrics in the style of your favorite artist by fine-tuning a GPT-2 model |  [Aleksey Korshuk](https://github.com/AlekseyKorshuk) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/AlekseyKorshuk/huggingartists/blob/master/huggingartists-demo.ipynb) |\n| [Train T5 in Tensorflow 2](https://github.com/snapthat/TF-T5-text-to-text) | How to train T5 for any task using Tensorflow 2. This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |\n| [Train T5 on TPU](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)  | How to train T5 on SQUAD with Transformers and Nlp | [Suraj Patil](https://github.com/patil-suraj) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil) |\n| [Fine-tune T5 for Classification and Multiple Choice](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb)  | How to fine-tune T5 for classification and multiple choice tasks using a text-to-text format with PyTorch Lightning |  [Suraj Patil](https://github.com/patil-suraj) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: e176cf9a-4957-4b2a-b08a-e46ef97ba437
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"e176cf9a-4957-4b2a-b08a-e46ef97ba437\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"19647714-3ba8-4877-b611-cd9c4bf5c093\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74757c3f59a27e93424cba24e98b9981afcfe44235c9057718462dbf88d7bba1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3679653f-11c9-4231-9051-f27b859643ee\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"03e4059e6cdb8d1f431fb51aa411f27bddccb5dc77a4ab65d7f877e7830585ac\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1669, \"end_char_idx\": 4102, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |\n| [Train T5 on TPU](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)  | How to train T5 on SQUAD with Transformers and Nlp | [Suraj Patil](https://github.com/patil-suraj) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil) |\n| [Fine-tune T5 for Classification and Multiple Choice](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb)  | How to fine-tune T5 for classification and multiple choice tasks using a text-to-text format with PyTorch Lightning |  [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) |\n| [Fine-tune DialoGPT on New Datasets and Languages](https://github.com/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb)  | How to fine-tune the DialoGPT model on a new dataset for open-dialog conversational chatbots |  [Nathan Cooper](https://github.com/ncoop57) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb) |\n| [Long Sequence Modeling with Reformer](https://github.com/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  | How to train on sequences as long as 500,000 tokens with Reformer |  [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  |\n| [Fine-tune BART for Summarization](https://github.com/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) | How to fine-tune BART for summarization with fastai using blurr | [Wayde Gilliam](https://ohmeow.com/) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 3679653f-11c9-4231-9051-f27b859643ee
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3679653f-11c9-4231-9051-f27b859643ee\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e176cf9a-4957-4b2a-b08a-e46ef97ba437\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0cac1fd67084d1022359bc1214802498c70abc02bed0969e29a13c0f44654ea3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7b435d1d-a546-485f-a366-ad3300ac95ef\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"43676eefa6ae10acac64ac2d310ae9b05be3d412612453e74c64b05b78b48f87\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3205, \"end_char_idx\": 5576, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb) |\n| [Long Sequence Modeling with Reformer](https://github.com/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  | How to train on sequences as long as 500,000 tokens with Reformer |  [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  |\n| [Fine-tune BART for Summarization](https://github.com/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) | How to fine-tune BART for summarization with fastai using blurr | [Wayde Gilliam](https://ohmeow.com/) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) |\n| [Fine-tune a pre-trained Transformer on anyone's tweets](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) | How to generate tweets in the style of your favorite Twitter account by fine-tuning a GPT-2 model |  [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) |\n| [Optimize \ud83e\udd17 Hugging Face models with Weights & Biases](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) | A complete tutorial showcasing W&B integration with Hugging Face | [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) |\n| [Pretrain Longformer](https://github.com/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb)  | How to build a \"long\" version of existing pretrained models |  [Iz Beltagy](https://beltagy.net) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 7b435d1d-a546-485f-a366-ad3300ac95ef
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"7b435d1d-a546-485f-a366-ad3300ac95ef\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3679653f-11c9-4231-9051-f27b859643ee\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d586fea41f6da6f0f3425c51bfce70fa736ac50c2babb49ec871734ac2f8d42d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f0f0e4cc-5799-42d1-9cbc-e3df232e8061\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0d93a5a7e3890ef8f99723966e997099817e5d7fbcd3ef21d0a2968a08340ae8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4634, \"end_char_idx\": 6882, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) |\n| [Optimize \ud83e\udd17 Hugging Face models with Weights & Biases](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) | A complete tutorial showcasing W&B integration with Hugging Face | [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) |\n| [Pretrain Longformer](https://github.com/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb)  | How to build a \"long\" version of existing pretrained models |  [Iz Beltagy](https://beltagy.net) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb) |\n| [Fine-tune Longformer for QA](https://github.com/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) | How to fine-tune longformer model for QA task | [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) |\n| [Evaluate Model with \ud83e\udd17nlp](https://github.com/patrickvonplaten/notebooks/blob/master/How_to_evaluate_Longformer_on_TriviaQA_using_NLP.ipynb) | How to evaluate longformer on TriviaQA with `nlp` | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1m7eTGlPmLRgoPkkA7rkhQdZ9ydpmsdLE?usp=sharing) |\n| [Fine-tune T5 for Sentiment Span Extraction](https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb)  | How to fine-tune T5 for sentiment span extraction using a text-to-text format with PyTorch Lightning |  [Lorenzo Ampil](https://github.com/enzoampil) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: f0f0e4cc-5799-42d1-9cbc-e3df232e8061
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f0f0e4cc-5799-42d1-9cbc-e3df232e8061\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7b435d1d-a546-485f-a366-ad3300ac95ef\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f42928e975e07076e62f5ece65bb8100d9c172af013b36309bafc4b47f7ad1e3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"30b6b27f-24ab-4ab4-b551-6ac719838f7e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4f6876d0a4c0ebf1f9e8b456eeef0cbe1df29ed19164be9f0eda6dc283b2f8dc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5576, \"end_char_idx\": 7882, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb) |\n| [Fine-tune Longformer for QA](https://github.com/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) | How to fine-tune longformer model for QA task | [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) |\n| [Evaluate Model with \ud83e\udd17nlp](https://github.com/patrickvonplaten/notebooks/blob/master/How_to_evaluate_Longformer_on_TriviaQA_using_NLP.ipynb) | How to evaluate longformer on TriviaQA with `nlp` | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1m7eTGlPmLRgoPkkA7rkhQdZ9ydpmsdLE?usp=sharing) |\n| [Fine-tune T5 for Sentiment Span Extraction](https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb)  | How to fine-tune T5 for sentiment span extraction using a text-to-text format with PyTorch Lightning |  [Lorenzo Ampil](https://github.com/enzoampil) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb) |\n| [Fine-tune DistilBert for Multiclass Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb) | How to fine-tune DistilBert for multiclass classification with PyTorch | [Abhishek Kumar Mishra](https://github.com/abhimishra91) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb)|\n|[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 30b6b27f-24ab-4ab4-b551-6ac719838f7e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"30b6b27f-24ab-4ab4-b551-6ac719838f7e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f0f0e4cc-5799-42d1-9cbc-e3df232e8061\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a9e0f6716bcc42f4ae1e91ab58b7e2f4ddbe9b5219411ba5edf691d7b3c48f5e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"199eba88-fec5-4f63-ad0a-fbe2b631ca70\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"907c9c21a581057ef6092e330668e407c94875911b4e0b8d8edfa29506e142a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6882, \"end_char_idx\": 9359, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb) |\n| [Fine-tune DistilBert for Multiclass Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb) | How to fine-tune DistilBert for multiclass classification with PyTorch | [Abhishek Kumar Mishra](https://github.com/abhimishra91) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb)|\n|[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|\n|[Fine-tune T5 for Summarization](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|How to fine-tune T5 for summarization in PyTorch and track experiments with WandB|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|\n|[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|\n|[Pretrain Reformer for Masked Language Modeling](https://github.com/patrickvonplaten/notebooks/blob/master/Reformer_For_Masked_LM.ipynb)| How to train a Reformer model with bi-directional self-attention layers | [Patrick von Platen](https://github.com/patrickvonplaten) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 199eba88-fec5-4f63-ad0a-fbe2b631ca70
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"199eba88-fec5-4f63-ad0a-fbe2b631ca70\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"30b6b27f-24ab-4ab4-b551-6ac719838f7e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b3ad9009bfdc68ae216debd2d5a68b1a9617ec528d54e4df93d55b343a656193\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cfe31154-8b93-46f8-b648-1b6610c40c7e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"85fa2245760f981b1f35660b7c49afa4abafb5fe617af081d9fb99ba38bfd74b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8377, \"end_char_idx\": 10803, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|\n|[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|\n|[Pretrain Reformer for Masked Language Modeling](https://github.com/patrickvonplaten/notebooks/blob/master/Reformer_For_Masked_LM.ipynb)| How to train a Reformer model with bi-directional self-attention layers | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1tzzh0i8PgDQGV3SMFUGxM7_gGae3K-uW?usp=sharing)|\n|[Expand and Fine Tune Sci-BERT](https://github.com/lordtt13/word-embeddings/blob/master/COVID-19%20Research%20Data/COVID-SciBERT.ipynb)| How to increase vocabulary of a pretrained SciBERT model from AllenAI on the CORD dataset and pipeline it. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1rqAR40goxbAfez1xvF3hBJphSCsvXmh8)|\n|[Fine Tune BlenderBotSmall for Summarization using the Trainer API](https://github.com/lordtt13/transformers-experiments/blob/master/Custom%20Tasks/fine-tune-blenderbot_small-for-summarization.ipynb)| How to fine-tune BlenderBotSmall for summarization on a custom dataset, using the Trainer API. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/19Wmupuls7mykSGyRN_Qo6lPQhgp56ymq?usp=sharing)|\n|[Fine-tune Electra and interpret with Integrated Gradients](https://github.com/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb) | How to fine-tune Electra for sentiment analysis and interpret predictions with Captum Integrated Gradients | [Eliza Szczechla](https://elsanns.github.io) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: cfe31154-8b93-46f8-b648-1b6610c40c7e
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"cfe31154-8b93-46f8-b648-1b6610c40c7e\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"199eba88-fec5-4f63-ad0a-fbe2b631ca70\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"6ea5efbb16cf1500241c36b30d05cb03219f587baa76820d8752040ad23f957f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3ac5e56d-5fcb-4836-a3db-d7accfc517bc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2bf0113b2a3c3b6e1ad1f33e5a921bb1700b307dfc6541bf459fe8dc67a0c88\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9767, \"end_char_idx\": 12337, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "| [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1rqAR40goxbAfez1xvF3hBJphSCsvXmh8)|\n|[Fine Tune BlenderBotSmall for Summarization using the Trainer API](https://github.com/lordtt13/transformers-experiments/blob/master/Custom%20Tasks/fine-tune-blenderbot_small-for-summarization.ipynb)| How to fine-tune BlenderBotSmall for summarization on a custom dataset, using the Trainer API. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/19Wmupuls7mykSGyRN_Qo6lPQhgp56ymq?usp=sharing)|\n|[Fine-tune Electra and interpret with Integrated Gradients](https://github.com/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb) | How to fine-tune Electra for sentiment analysis and interpret predictions with Captum Integrated Gradients | [Eliza Szczechla](https://elsanns.github.io) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb)|\n|[fine-tune a non-English GPT-2 Model with Trainer class](https://github.com/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb) | How to fine-tune a non-English GPT-2 Model with Trainer class | [Philipp Schmid](https://www.philschmid.de) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb)|\n|[Fine-tune a DistilBERT Model for Multi Label Classification task](https://github.com/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb) | How to fine-tune a DistilBERT Model for Multi Label Classification task | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb)|\n|[Fine-tune ALBERT for sentence-pair classification](https://github.com/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb) | How to fine-tune an ALBERT model or another BERT-based model for the sentence-pair classification task | [Nadir El Manouzi](https://github.com/NadirEM) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 3ac5e56d-5fcb-4836-a3db-d7accfc517bc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3ac5e56d-5fcb-4836-a3db-d7accfc517bc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cfe31154-8b93-46f8-b648-1b6610c40c7e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"48847e1090dbe7503e4bbe3262eb3c518c13ec6378b6668a0c109e4d29f78ae3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"74776684-ca80-479a-aa5b-0b6c82ce81e9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"81394bed4305e0d14d84fad18b41452ea7a31658c6b5493e6766e1f856ab11eb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11293, \"end_char_idx\": 13629, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb)|\n|[Fine-tune a DistilBERT Model for Multi Label Classification task](https://github.com/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb) | How to fine-tune a DistilBERT Model for Multi Label Classification task | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb)|\n|[Fine-tune ALBERT for sentence-pair classification](https://github.com/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb) | How to fine-tune an ALBERT model or another BERT-based model for the sentence-pair classification task | [Nadir El Manouzi](https://github.com/NadirEM) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb)|\n|[Fine-tune Roberta for sentiment analysis](https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb) | How to fine-tune a Roberta model for sentiment analysis | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb)|\n|[Evaluating Question Generation Models](https://github.com/flexudy-pipe/qugeev) | How accurate are the answers to questions generated by your seq2seq transformer model? | [Pascal Zoleko](https://github.com/zolekode) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bpsSqCQU-iw_5nNoRm_crPq6FRuJthq_?usp=sharing)|\n|[Classify text with DistilBERT and Tensorflow](https://github.com/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb) | How to fine-tune DistilBERT for text classification in TensorFlow | [Peter Bayerle](https://github.com/peterbayerle) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 74776684-ca80-479a-aa5b-0b6c82ce81e9
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"74776684-ca80-479a-aa5b-0b6c82ce81e9\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3ac5e56d-5fcb-4836-a3db-d7accfc517bc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"55cb64c04f0c9143458b64aeaa0ac09c338dfc9e1301597619a59d98cf97f83a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0a4225ca-44d5-4954-a2fd-a2f434392276\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4d1b70bf401d7b8d1835e95f85b9c6c7c7c3a5c66cce3eefc058e29e42fc239c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12337, \"end_char_idx\": 14703, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb)|\n|[Fine-tune Roberta for sentiment analysis](https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb) | How to fine-tune a Roberta model for sentiment analysis | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb)|\n|[Evaluating Question Generation Models](https://github.com/flexudy-pipe/qugeev) | How accurate are the answers to questions generated by your seq2seq transformer model? | [Pascal Zoleko](https://github.com/zolekode) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bpsSqCQU-iw_5nNoRm_crPq6FRuJthq_?usp=sharing)|\n|[Classify text with DistilBERT and Tensorflow](https://github.com/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb) | How to fine-tune DistilBERT for text classification in TensorFlow | [Peter Bayerle](https://github.com/peterbayerle) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb)|\n|[Leverage BERT for Encoder-Decoder Summarization on CNN/Dailymail](https://github.com/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb) | How to warm-start a *EncoderDecoderModel* with a *google-bert/bert-base-uncased* checkpoint for summarization on CNN/Dailymail | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb)|\n|[Leverage RoBERTa for Encoder-Decoder Summarization on BBC XSum](https://github.com/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb) | How to warm-start a shared *EncoderDecoderModel* with a *FacebookAI/roberta-base* checkpoint for summarization on BBC/XSum | [Patrick von Platen](https://github.com/patrickvonplaten) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 0a4225ca-44d5-4954-a2fd-a2f434392276
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0a4225ca-44d5-4954-a2fd-a2f434392276\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"74776684-ca80-479a-aa5b-0b6c82ce81e9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0c3e7de539006203d02e2e493c671c0fc62eda424de1330a4453386a848b97be\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"17615da8-5d33-4460-a411-4ffebb331b07\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9d8ffb1ae73fff6d8f6498f4c12d81e2909193bda9968544980adea14993260b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13629, \"end_char_idx\": 15876, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb)|\n|[Leverage BERT for Encoder-Decoder Summarization on CNN/Dailymail](https://github.com/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb) | How to warm-start a *EncoderDecoderModel* with a *google-bert/bert-base-uncased* checkpoint for summarization on CNN/Dailymail | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb)|\n|[Leverage RoBERTa for Encoder-Decoder Summarization on BBC XSum](https://github.com/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb) | How to warm-start a shared *EncoderDecoderModel* with a *FacebookAI/roberta-base* checkpoint for summarization on BBC/XSum | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb)|\n|[Fine-tune TAPAS on Sequential Question Answering (SQA)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb) | How to fine-tune *TapasForQuestionAnswering* with a *tapas-base* checkpoint on the Sequential Question Answering (SQA) dataset | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb)|\n|[Evaluate TAPAS on Table Fact Checking (TabFact)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb) | How to evaluate a fine-tuned *TapasForSequenceClassification* with a *tapas-base-finetuned-tabfact* checkpoint using a combination of the \ud83e\udd17 datasets and \ud83e\udd17 transformers libraries | [Niels Rogge](https://github.com/nielsrogge) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 17615da8-5d33-4460-a411-4ffebb331b07
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"17615da8-5d33-4460-a411-4ffebb331b07\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0a4225ca-44d5-4954-a2fd-a2f434392276\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9147ae27054d6af462ef8b943d5525823b7388d22adb2a75db88f5c8d36c85c1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"052038f9-fc1b-4e90-91f8-7db503be69e6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2ac6a16f6b7109db2f03ad1df0552e6370961d89eee34ed421acfcb26efbd4d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14703, \"end_char_idx\": 17436, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb)|\n|[Fine-tune TAPAS on Sequential Question Answering (SQA)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb) | How to fine-tune *TapasForQuestionAnswering* with a *tapas-base* checkpoint on the Sequential Question Answering (SQA) dataset | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb)|\n|[Evaluate TAPAS on Table Fact Checking (TabFact)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb) | How to evaluate a fine-tuned *TapasForSequenceClassification* with a *tapas-base-finetuned-tabfact* checkpoint using a combination of the \ud83e\udd17 datasets and \ud83e\udd17 transformers libraries | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb)|\n|[Fine-tuning mBART for translation](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb) | How to fine-tune mBART using Seq2SeqTrainer for Hindi to English translation | [Vasudev Gupta](https://github.com/vasudevgupta7) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb)|\n|[Fine-tune LayoutLM on FUNSD (a form understanding dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb) | How to fine-tune *LayoutLMForTokenClassification* on the FUNSD dataset for information extraction from scanned documents | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb)|\n|[Fine-Tune DistilGPT2 and Generate Text](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb) | How to fine-tune DistilGPT2 and generate text | [Aakash Tripathi](https://github.com/tripathiaakash) | [!"}


created_at: 2024-08-26 20:30:03
topic: persistent://default/default/621deaf2-9359-4ac4-ada0-047176703e13
id: 052038f9-fc1b-4e90-91f8-7db503be69e6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"052038f9-fc1b-4e90-91f8-7db503be69e6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"37cbed30-c667-4ebd-96e9-00c55d9acc02\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"17615da8-5d33-4460-a411-4ffebb331b07\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e7a4a68c89db29749dfa854d37ea870f0735b81b95ca660d59332132e94d5e4f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8c7a9b4d-ba33-4f07-8810-8f20a897ff40\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5ad18b33ca524874f40ea2c8c82314066533866bf20837faf9555c311f9123ac\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16380, \"end_char_idx\": 18946, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "ref_doc_id": "37cbed30-c667-4ebd-96e9-00c55d9acc02", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb)|\n|[Fine-tune LayoutLM on FUNSD (a form understanding dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb) | How to fine-tune *LayoutLMForTokenClassification* on the FUNSD dataset for information extraction from scanned documents | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb)|\n|[Fine-Tune DistilGPT2 and Generate Text](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb) | How to fine-tune DistilGPT2 and generate text | [Aakash Tripathi](https://github.com/tripathiaakash) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb)|\n|[Fine-Tune LED on up to 8K tokens](https://github.com/patrickvonplaten/notebooks/blob/master/Fine_tune_Longformer_Encoder_Decoder_(LED)_for_Summarization_on_pubmed.ipynb) | How to fine-tune LED on pubmed for long-range summarization | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_tune_Longformer_Encoder_Decoder_(LED)_for_Summarization_on_pubmed.ipynb)|\n|[Evaluate LED on Arxiv](https://github.com/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb) | How to effectively evaluate LED on long-range summarization | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb)|\n|[Fine-tune LayoutLM on RVL-CDIP (a document image classification dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForSequenceClassification_on_RVL_CDIP.ipynb) | How to fine-tune *LayoutLMForSequenceClassification* on the RVL-CDIP dataset for scanned document classification | [Niels Rogge](https://github.com/nielsrogge) | [!"}


