# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: sysdb
filename: 00005-remove-topic.sqlite.sql
hash: b1367c826b8fba5f96f27befdc1d42d2
sql:
-- Remove the topic column from the Collections and Segments tables

ALTER TABLE collections DROP COLUMN topic;
ALTER TABLE segments DROP COLUMN topic;


dir: sysdb
filename: 00006-collection-segment-metadata.sqlite.sql
hash: 4eea7468935bf25d4604a0fed2366116
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE collection_metadata ADD COLUMN bool_value INTEGER;
ALTER TABLE segment_metadata ADD COLUMN bool_value INTEGER;


dir: sysdb
filename: 00007-collection-config.sqlite.sql
hash: 1c7e63bba346a42a18b6ab7f1c989bed
sql:
-- Stores collection configuration dictionaries.
ALTER TABLE collections ADD COLUMN config_json_str TEXT;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 83800250-89f6-4012-89dc-ecda912192ca
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"83800250-89f6-4012-89dc-ecda912192ca\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0977e3d2-4f04-4b40-8607-875fe1f5ddb1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7b6e926217884aa4d45d293ae4066739adfb48fddce666f2b7ac2b49e5479503\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2903, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "# Preprocess\n\n[[open-in-colab]]\n\nBefore you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \ud83e\udd17 Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you'll learn that for:\n\n* Text, use a [Tokenizer](./main_classes/tokenizer) to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors.\n* Speech and audio, use a [Feature extractor](./main_classes/feature_extractor) to extract sequential features from audio waveforms and convert them into tensors.\n* Image inputs use a [ImageProcessor](./main_classes/image_processor) to convert images into tensors.\n* Multimodal inputs, use a [Processor](./main_classes/processors) to combine a tokenizer and a feature extractor or image processor.\n\n<Tip>\n\n`AutoProcessor` **always** works and automatically chooses the correct class for the model you're using, whether you're using a tokenizer, image processor, feature extractor or processor.\n\n</Tip>\n\nBefore you begin, install \ud83e\udd17 Datasets so you can load some datasets to experiment with:\n\n```bash\npip install datasets\n```\n\n## Natural Language Processing\n\n<Youtube id=\"Yffk5aydLzg\"/>\n\nThe main tool for preprocessing textual data is a [tokenizer](main_classes/tokenizer). A tokenizer splits text into *tokens* according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer.\n\n<Tip>\n\nIf you plan on using a pretrained model, it's important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the *vocab*) during pretraining.\n\n</Tip>\n\nGet started by loading a pretrained tokenizer with the [`AutoTokenizer.from_pretrained`] method. This downloads the *vocab* a model was pretrained with:\n\n```py\n>>> from transformers import AutoTokenizer\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-cased\")\n```\n\nThen pass your text to the tokenizer:\n\n```py\n>>> encoded_input = tokenizer(\"Do not meddle in the affairs of wizards, for they are subtle and quick to anger.\")\n>>> print(encoded_input)\n{'input_ids': [101, 2079, 2025, 19960, 10362, 1999, 1996, 3821, 1997, 16657, 1010, 2005, 2027, 2024, 11259, 1998, 4248, 2000, 4963, 1012, 102],\n 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n```\n\nThe tokenizer returns a dictionary with three important items:\n\n* [input_ids](glossary#input-ids) are the indices corresponding to each token in the sentence."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 0977e3d2-4f04-4b40-8607-875fe1f5ddb1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0977e3d2-4f04-4b40-8607-875fe1f5ddb1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"83800250-89f6-4012-89dc-ecda912192ca\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cb33f20b07ca1a9f80777824548579572d967acce85051e22799c63e763bccb9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5e97d5b5-574b-4118-b47e-4891aefd1937\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8336be552eec8e5c8f393668167fc2d4ec77acc072931d5e0e258f60c419c4c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1773, \"end_char_idx\": 3814, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referred to as the *vocab*) during pretraining.\n\n</Tip>\n\nGet started by loading a pretrained tokenizer with the [`AutoTokenizer.from_pretrained`] method. This downloads the *vocab* a model was pretrained with:\n\n```py\n>>> from transformers import AutoTokenizer\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-cased\")\n```\n\nThen pass your text to the tokenizer:\n\n```py\n>>> encoded_input = tokenizer(\"Do not meddle in the affairs of wizards, for they are subtle and quick to anger.\")\n>>> print(encoded_input)\n{'input_ids': [101, 2079, 2025, 19960, 10362, 1999, 1996, 3821, 1997, 16657, 1010, 2005, 2027, 2024, 11259, 1998, 4248, 2000, 4963, 1012, 102],\n 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n```\n\nThe tokenizer returns a dictionary with three important items:\n\n* [input_ids](glossary#input-ids) are the indices corresponding to each token in the sentence.\n* [attention_mask](glossary#attention-mask) indicates whether a token should be attended to or not.\n* [token_type_ids](glossary#token-type-ids) identifies which sequence a token belongs to when there is more than one sequence.\n\nReturn your input by decoding the `input_ids`:\n\n```py\n>>> tokenizer.decode(encoded_input[\"input_ids\"])\n'[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]'\n```\n\nAs you can see, the tokenizer added two special tokens - `CLS` and `SEP` (classifier and separator) - to the sentence. Not all models need\nspecial tokens, but if they do, the tokenizer automatically adds them for you.\n\nIf there are several sentences you want to preprocess, pass them as a list to the tokenizer:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 5e97d5b5-574b-4118-b47e-4891aefd1937
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5e97d5b5-574b-4118-b47e-4891aefd1937\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0977e3d2-4f04-4b40-8607-875fe1f5ddb1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"82dae28fc3d6d06948bcc12f0153a5e5ba9366d4de83ca10c306720f9c9ff264\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2c1719d8-a163-4e70-a5e5-5364896d7d05\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bd9f6313951e6714e51bc15e1f90fac0aac4da7a8e937e5b66bb62961712c8b5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2904, \"end_char_idx\": 4970, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "* [attention_mask](glossary#attention-mask) indicates whether a token should be attended to or not.\n* [token_type_ids](glossary#token-type-ids) identifies which sequence a token belongs to when there is more than one sequence.\n\nReturn your input by decoding the `input_ids`:\n\n```py\n>>> tokenizer.decode(encoded_input[\"input_ids\"])\n'[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]'\n```\n\nAs you can see, the tokenizer added two special tokens - `CLS` and `SEP` (classifier and separator) - to the sentence. Not all models need\nspecial tokens, but if they do, the tokenizer automatically adds them for you.\n\nIf there are several sentences you want to preprocess, pass them as a list to the tokenizer:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_inputs = tokenizer(batch_sentences)\n>>> print(encoded_inputs)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1]]}\n```\n\n### Pad\n\nSentences aren't always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special *padding token* to shorter sentences.\n\nSet the `padding` parameter to `True` to pad the shorter sequences in the batch to match the longest sequence:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2c1719d8-a163-4e70-a5e5-5364896d7d05
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2c1719d8-a163-4e70-a5e5-5364896d7d05\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5e97d5b5-574b-4118-b47e-4891aefd1937\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"7724906a002dc54e0aeb7f8ce7e2b11bb1633192a02d1b3893a62a537ccd5151\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"68e0243d-7b6a-4d8e-812a-166863167c03\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3250eb6b2bb63bb28fbd62d8853a0e87b7f5aa26e1ebcc850c8a2524edb5198e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4574, \"end_char_idx\": 6283, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "Padding is a strategy for ensuring tensors are rectangular by adding a special *padding token* to shorter sentences.\n\nSet the `padding` parameter to `True` to pad the shorter sequences in the batch to match the longest sequence:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True)\n>>> print(encoded_input)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]]}\n```\n\nThe first and third sentences are now padded with `0`'s because they are shorter.\n\n### Truncation\n\nOn the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you'll need to truncate the sequence to a shorter length.\n\nSet the `truncation` parameter to `True` to truncate a sequence to the maximum length accepted by the model:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 68e0243d-7b6a-4d8e-812a-166863167c03
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"68e0243d-7b6a-4d8e-812a-166863167c03\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2c1719d8-a163-4e70-a5e5-5364896d7d05\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1505b58ef2d673e96cfe0eecea44a21fc3469d6562e8197937d906ba622d23d3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"211197ec-0ef8-4818-af5b-1e1fead26b3d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f34879afe7db6f923540c8fe90820cb6a8b79ffca3281adadf5751f16a091d58\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5824, \"end_char_idx\": 7603, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "### Truncation\n\nOn the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you'll need to truncate the sequence to a shorter length.\n\nSet the `truncation` parameter to `True` to truncate a sequence to the maximum length accepted by the model:\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)\n>>> print(encoded_input)\n{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n               [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]],\n 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                    [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]]}\n```\n\n<Tip>\n\nCheck out the [Padding and truncation](./pad_truncation) concept guide to learn more different padding and truncation arguments.\n\n</Tip>\n\n### Build tensors\n\nFinally, you want the tokenizer to return the actual tensors that get fed to the model.\n\nSet the `return_tensors` parameter to either `pt` for PyTorch, or `tf` for TensorFlow:\n\n<frameworkcontent>\n<pt>\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 211197ec-0ef8-4818-af5b-1e1fead26b3d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"211197ec-0ef8-4818-af5b-1e1fead26b3d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"68e0243d-7b6a-4d8e-812a-166863167c03\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"bee24609fcad865fd1e0cd04c39f87a6448e9eb57ee230b0f18342df56df06c8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"822d5206-5fe2-4bdd-8ecb-f1999f197002\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e8b58e4f6ef3714c1b8d88ab0ceb04951452251409f5513703a567bbcb65143b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7208, \"end_char_idx\": 8654, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "</Tip>\n\n### Build tensors\n\nFinally, you want the tokenizer to return the actual tensors that get fed to the model.\n\nSet the `return_tensors` parameter to either `pt` for PyTorch, or `tf` for TensorFlow:\n\n<frameworkcontent>\n<pt>\n\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors=\"pt\")\n>>> print(encoded_input)\n{'input_ids': tensor([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n                      [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n                      [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]]),\n 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                           [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n                           [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]),\n 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n                           [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n                           [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}\n```\n</pt>\n<tf>\n```py\n>>> batch_sentences = [\n...     \"But what about second breakfast?\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 822d5206-5fe2-4bdd-8ecb-f1999f197002
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"822d5206-5fe2-4bdd-8ecb-f1999f197002\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"211197ec-0ef8-4818-af5b-1e1fead26b3d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"475eb98a98a46cefa9f739543358037cf734ac45598a1fa9e5a79ec7b6366342\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0f95990c-43a1-4552-ae03-00f8492677ac\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8318013a55868ead22fceda3515200ae8857fa4a7a892b687c376aa66ed8975a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8559, \"end_char_idx\": 10249, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "\",\n...     \"Don't think he knows about second breakfast, Pip.\",\n...     \"What about elevensies?\",\n... ]\n>>> encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors=\"tf\")\n>>> print(encoded_input)\n{'input_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0],\n       [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102],\n       [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]],\n      dtype=int32)>,\n 'token_type_ids': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],\n       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>,\n 'attention_mask': <tf.Tensor: shape=(2, 9), dtype=int32, numpy=\narray([[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],\n       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n       [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=int32)>}\n```\n</tf>\n</frameworkcontent>\n\n<Tip>\nDifferent pipelines support tokenizer arguments in their `__call__()` differently. `text-2-text-generation` pipelines support (i.e. pass on)\nonly `truncation`. `text-generation` pipelines support `max_length`, `truncation`, `padding` and `add_special_tokens`. \nIn `fill-mask` pipelines, tokenizer arguments can be passed in the `tokenizer_kwargs` argument (dictionary).\n</Tip>\n\n## Audio\n\nFor audio tasks, you'll need a [feature extractor](main_classes/feature_extractor) to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 0f95990c-43a1-4552-ae03-00f8492677ac
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0f95990c-43a1-4552-ae03-00f8492677ac\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"822d5206-5fe2-4bdd-8ecb-f1999f197002\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f50103c82d1802a67e0d4d5b02df3542e0a16aa10831d0e848bbd6c073431316\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"46fe234a-0d10-4a6e-b8e2-6025decd5fb0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4b45e3f324520fefa6e70718e76fad19d056612febb742f115eaa857818f8e68\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9717, \"end_char_idx\": 12006, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "`text-2-text-generation` pipelines support (i.e. pass on)\nonly `truncation`. `text-generation` pipelines support `max_length`, `truncation`, `padding` and `add_special_tokens`. \nIn `fill-mask` pipelines, tokenizer arguments can be passed in the `tokenizer_kwargs` argument (dictionary).\n</Tip>\n\n## Audio\n\nFor audio tasks, you'll need a [feature extractor](main_classes/feature_extractor) to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors.\n\nLoad the [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:\n\n```py\n>>> from datasets import load_dataset, Audio\n\n>>> dataset = load_dataset(\"PolyAI/minds14\", name=\"en-US\", split=\"train\")\n```\n\nAccess the first element of the `audio` column to take a look at the input. Calling the `audio` column automatically loads and resamples the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 0.        ,  0.00024414, -0.00024414, ..., -0.00024414,\n         0.        ,  0.        ], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 8000}\n```\n\nThis returns three items:\n\n* `array` is the speech signal loaded - and potentially resampled - as a 1D array.\n* `path` points to the location of the audio file.\n* `sampling_rate` refers to how many data points in the speech signal are measured per second.\n\nFor this tutorial, you'll use the [Wav2Vec2](https://huggingface.co/facebook/wav2vec2-base) model. Take a look at the model card, and you'll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data's sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 46fe234a-0d10-4a6e-b8e2-6025decd5fb0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"46fe234a-0d10-4a6e-b8e2-6025decd5fb0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0f95990c-43a1-4552-ae03-00f8492677ac\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eae12348a62d278c14a344b2dae0214b11cd38c6ee262077711804e2e917ebff\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f7b75cc9-2658-4a30-9ed4-ae57cb4af0e0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0ede54a9d0ee9936ac77410e7cd1850768f9353bcf60dcc8f7575615d2e6e736\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10720, \"end_char_idx\": 13097, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "Calling the `audio` column automatically loads and resamples the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 0.        ,  0.00024414, -0.00024414, ..., -0.00024414,\n         0.        ,  0.        ], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 8000}\n```\n\nThis returns three items:\n\n* `array` is the speech signal loaded - and potentially resampled - as a 1D array.\n* `path` points to the location of the audio file.\n* `sampling_rate` refers to how many data points in the speech signal are measured per second.\n\nFor this tutorial, you'll use the [Wav2Vec2](https://huggingface.co/facebook/wav2vec2-base) model. Take a look at the model card, and you'll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data's sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2. Call the `audio` column again to resample the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 2.3443763e-05,  2.1729663e-04,  2.2145823e-04, ...,\n         3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 16000}\n```\n\nNext, load a feature extractor to normalize and pad the input. When padding textual data, a `0` is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a `0` - interpreted as silence - to `array`.\n\nLoad the feature extractor with [`AutoFeatureExtractor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoFeatureExtractor\n\n>>> feature_extractor = AutoFeatureExtractor.from_pretrained(\"facebook/wav2vec2-base\")\n```\n\nPass the audio `array` to the feature extractor. We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f7b75cc9-2658-4a30-9ed4-ae57cb4af0e0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f7b75cc9-2658-4a30-9ed4-ae57cb4af0e0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"46fe234a-0d10-4a6e-b8e2-6025decd5fb0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a38cbad9766cb401559c034fef6a25d91aed93818efca515ee56c30d7e1de8bf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"61b88408-bb84-4477-8e2e-8c854a3eb243\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ce4b82e7e4557e91950b591b48945e9f46d177f24a96da01936e1f6dbe87dfb7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11737, \"end_char_idx\": 14442, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "If your data's sampling rate isn't the same, then you need to resample your data.\n\n1. Use \ud83e\udd17 Datasets' [`~datasets.Dataset.cast_column`] method to upsample the sampling rate to 16kHz:\n\n```py\n>>> dataset = dataset.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\n2. Call the `audio` column again to resample the audio file:\n\n```py\n>>> dataset[0][\"audio\"]\n{'array': array([ 2.3443763e-05,  2.1729663e-04,  2.2145823e-04, ...,\n         3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',\n 'sampling_rate': 16000}\n```\n\nNext, load a feature extractor to normalize and pad the input. When padding textual data, a `0` is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a `0` - interpreted as silence - to `array`.\n\nLoad the feature extractor with [`AutoFeatureExtractor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoFeatureExtractor\n\n>>> feature_extractor = AutoFeatureExtractor.from_pretrained(\"facebook/wav2vec2-base\")\n```\n\nPass the audio `array` to the feature extractor. We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur.\n\n```py\n>>> audio_input = [dataset[0][\"audio\"][\"array\"]]\n>>> feature_extractor(audio_input, sampling_rate=16000)\n{'input_values': [array([ 3.8106556e-04,  2.7506407e-03,  2.8015103e-03, ...,\n        5.6335266e-04,  4.6588284e-06, -1.7142107e-04], dtype=float32)]}\n```\n\nJust like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:\n\n```py\n>>> dataset[0][\"audio\"][\"array\"].shape\n(173398,)\n\n>>> dataset[1][\"audio\"][\"array\"].shape\n(106496,)\n```\n\nCreate a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:\n\n```py\n>>> def preprocess_function(examples):\n...     audio_arrays = [x[\"array\"] for x in examples[\"audio\"]]\n...     inputs = feature_extractor(\n...         audio_arrays,\n...         sampling_rate=16000,\n...         padding=True,\n...         max_length=100000,\n...         truncation=True,\n...     )\n...     return inputs\n```\n\nApply the `preprocess_function` to the first few examples in the dataset:\n\n```py\n>>> processed_dataset = preprocess_function(dataset[:5])\n```\n\nThe sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 61b88408-bb84-4477-8e2e-8c854a3eb243
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"61b88408-bb84-4477-8e2e-8c854a3eb243\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f7b75cc9-2658-4a30-9ed4-ae57cb4af0e0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d9cb780779a56717491def3d88e478e65eb7ff6895cd3d9c7b68beb69f32f05b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7322035f-1a98-468f-8864-e44859db95b2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8babeb15154b929a74c648ac374258ccbd38ffb97d59a18506a171204f37c0a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12960, \"end_char_idx\": 16191, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "We also recommend adding the `sampling_rate` argument in the feature extractor in order to better debug any silent errors that may occur.\n\n```py\n>>> audio_input = [dataset[0][\"audio\"][\"array\"]]\n>>> feature_extractor(audio_input, sampling_rate=16000)\n{'input_values': [array([ 3.8106556e-04,  2.7506407e-03,  2.8015103e-03, ...,\n        5.6335266e-04,  4.6588284e-06, -1.7142107e-04], dtype=float32)]}\n```\n\nJust like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:\n\n```py\n>>> dataset[0][\"audio\"][\"array\"].shape\n(173398,)\n\n>>> dataset[1][\"audio\"][\"array\"].shape\n(106496,)\n```\n\nCreate a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:\n\n```py\n>>> def preprocess_function(examples):\n...     audio_arrays = [x[\"array\"] for x in examples[\"audio\"]]\n...     inputs = feature_extractor(\n...         audio_arrays,\n...         sampling_rate=16000,\n...         padding=True,\n...         max_length=100000,\n...         truncation=True,\n...     )\n...     return inputs\n```\n\nApply the `preprocess_function` to the first few examples in the dataset:\n\n```py\n>>> processed_dataset = preprocess_function(dataset[:5])\n```\n\nThe sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!\n\n```py\n>>> processed_dataset[\"input_values\"][0].shape\n(100000,)\n\n>>> processed_dataset[\"input_values\"][1].shape\n(100000,)\n```\n\n## Computer vision\n\nFor computer vision tasks, you'll need an [image processor](main_classes/image_processor) to prepare your dataset for the model.\nImage preprocessing consists of several steps that convert images into the input expected by the model. These steps\ninclude but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.\n\n<Tip>\n\nImage preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation\ntransform image data, but they serve different purposes:\n\n* Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations.\n* Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained.\n\nYou can use any library you like for image augmentation. For image preprocessing, use the `ImageProcessor` associated with the model.\n\n</Tip>\n\nLoad the [food101](https://huggingface.co/datasets/food101) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:\n\n<Tip>\n\nUse \ud83e\udd17 Datasets `split` parameter to only load a small sample from the training split since the dataset is quite large!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 7322035f-1a98-468f-8864-e44859db95b2
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"7322035f-1a98-468f-8864-e44859db95b2\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"61b88408-bb84-4477-8e2e-8c854a3eb243\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"77c133edd86254b4e6604a73f6651b639bf1bec5beae8a8d44ae1df6aedd9800\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2fb8973d-0d9e-42c5-b3e2-6c51526eb530\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1410408978c9a36806a714f31d670fdc4c6466b75f566cf9734810c5d6fd0b74\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14389, \"end_char_idx\": 17918, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "You can pass your processed dataset to the model now!\n\n```py\n>>> processed_dataset[\"input_values\"][0].shape\n(100000,)\n\n>>> processed_dataset[\"input_values\"][1].shape\n(100000,)\n```\n\n## Computer vision\n\nFor computer vision tasks, you'll need an [image processor](main_classes/image_processor) to prepare your dataset for the model.\nImage preprocessing consists of several steps that convert images into the input expected by the model. These steps\ninclude but are not limited to resizing, normalizing, color channel correction, and converting images to tensors.\n\n<Tip>\n\nImage preprocessing often follows some form of image augmentation. Both image preprocessing and image augmentation\ntransform image data, but they serve different purposes:\n\n* Image augmentation alters images in a way that can help prevent overfitting and increase the robustness of the model. You can get creative in how you augment your data - adjust brightness and colors, crop, rotate, resize, zoom, etc. However, be mindful not to change the meaning of the images with your augmentations.\n* Image preprocessing guarantees that the images match the model\u2019s expected input format. When fine-tuning a computer vision model, images must be preprocessed exactly as when the model was initially trained.\n\nYou can use any library you like for image augmentation. For image preprocessing, use the `ImageProcessor` associated with the model.\n\n</Tip>\n\nLoad the [food101](https://huggingface.co/datasets/food101) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:\n\n<Tip>\n\nUse \ud83e\udd17 Datasets `split` parameter to only load a small sample from the training split since the dataset is quite large!\n\n</Tip>\n\n```py\n>>> from datasets import load_dataset\n\n>>> dataset = load_dataset(\"food101\", split=\"train[:100]\")\n```\n\nNext, take a look at the image with \ud83e\udd17 Datasets [`Image`](https://huggingface.co/docs/datasets/package_reference/main_classes?highlight=image#datasets.Image) feature:\n\n```py\n>>> dataset[0][\"image\"]\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png\"/>\n</div>\n\nLoad the image processor with [`AutoImageProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoImageProcessor\n\n>>> image_processor = AutoImageProcessor.from_pretrained(\"google/vit-base-patch16-224\")\n```\n\nFirst, let's add some image augmentation. You can use any library you prefer, but in this tutorial, we'll use torchvision's [`transforms`](https://pytorch.org/vision/stable/transforms.html) module. If you're interested in using another data augmentation library, learn how in the [Albumentations](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) or [Kornia notebooks](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb).\n\n1. Here we use [`Compose`](https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html) to chain together a couple of\ntransforms - [`RandomResizedCrop`](https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html) and [`ColorJitter`](https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html).\nNote that for resizing, we can get the image size requirements from the `image_processor`."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2fb8973d-0d9e-42c5-b3e2-6c51526eb530
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2fb8973d-0d9e-42c5-b3e2-6c51526eb530\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7322035f-1a98-468f-8864-e44859db95b2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"3e67c67e9fbdc5404f93b4c31ac5eef19f2306d0de16e8650c0900d73ee6ed1b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f8ccfd99-88e0-470a-becc-16d7ed229a77\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4ba95b02d45eb167035a2bb2d6ac07c2f79468a67297cc350a224c5b918ecd96\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16193, \"end_char_idx\": 19543, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "</Tip>\n\n```py\n>>> from datasets import load_dataset\n\n>>> dataset = load_dataset(\"food101\", split=\"train[:100]\")\n```\n\nNext, take a look at the image with \ud83e\udd17 Datasets [`Image`](https://huggingface.co/docs/datasets/package_reference/main_classes?highlight=image#datasets.Image) feature:\n\n```py\n>>> dataset[0][\"image\"]\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png\"/>\n</div>\n\nLoad the image processor with [`AutoImageProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoImageProcessor\n\n>>> image_processor = AutoImageProcessor.from_pretrained(\"google/vit-base-patch16-224\")\n```\n\nFirst, let's add some image augmentation. You can use any library you prefer, but in this tutorial, we'll use torchvision's [`transforms`](https://pytorch.org/vision/stable/transforms.html) module. If you're interested in using another data augmentation library, learn how in the [Albumentations](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb) or [Kornia notebooks](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb).\n\n1. Here we use [`Compose`](https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html) to chain together a couple of\ntransforms - [`RandomResizedCrop`](https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html) and [`ColorJitter`](https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html).\nNote that for resizing, we can get the image size requirements from the `image_processor`. For some models, an exact height and\nwidth are expected, for others only the `shortest_edge` is defined.\n\n```py\n>>> from torchvision.transforms import RandomResizedCrop, ColorJitter, Compose\n\n>>> size = (\n...     image_processor.size[\"shortest_edge\"]\n...     if \"shortest_edge\" in image_processor.size\n...     else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n... )\n\n>>> _transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5)])\n```\n\n2. The model accepts [`pixel_values`](model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel.forward.pixel_values)\nas its input. `ImageProcessor` can take care of normalizing the images, and generating appropriate tensors.\nCreate a function that combines image augmentation and image preprocessing for a batch of images and generates `pixel_values`:\n\n```py\n>>> def transforms(examples):\n...     images = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n...     examples[\"pixel_values\"] = image_processor(images, do_resize=False, return_tensors=\"pt\")[\"pixel_values\"]\n...     return examples\n```\n\n<Tip>\n\nIn the example above we set `do_resize=False` because we have already resized the images in the image augmentation transformation,\nand leveraged the `size` attribute from the appropriate `image_processor`. If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f8ccfd99-88e0-470a-becc-16d7ed229a77
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f8ccfd99-88e0-470a-becc-16d7ed229a77\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2fb8973d-0d9e-42c5-b3e2-6c51526eb530\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"bbc6e5cc6f03d73e736e09800cd410f2ce36b9467f71570aeda11642bb0a7138\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c6b5baf8-a449-4a34-aad5-dc6df4f7905c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dd0abd2823e1c334ee40889278d1ca109fe21917312dd7f23cf17b401dd8e964\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17828, \"end_char_idx\": 20921, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "Note that for resizing, we can get the image size requirements from the `image_processor`. For some models, an exact height and\nwidth are expected, for others only the `shortest_edge` is defined.\n\n```py\n>>> from torchvision.transforms import RandomResizedCrop, ColorJitter, Compose\n\n>>> size = (\n...     image_processor.size[\"shortest_edge\"]\n...     if \"shortest_edge\" in image_processor.size\n...     else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n... )\n\n>>> _transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5)])\n```\n\n2. The model accepts [`pixel_values`](model_doc/vision-encoder-decoder#transformers.VisionEncoderDecoderModel.forward.pixel_values)\nas its input. `ImageProcessor` can take care of normalizing the images, and generating appropriate tensors.\nCreate a function that combines image augmentation and image preprocessing for a batch of images and generates `pixel_values`:\n\n```py\n>>> def transforms(examples):\n...     images = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n...     examples[\"pixel_values\"] = image_processor(images, do_resize=False, return_tensors=\"pt\")[\"pixel_values\"]\n...     return examples\n```\n\n<Tip>\n\nIn the example above we set `do_resize=False` because we have already resized the images in the image augmentation transformation,\nand leveraged the `size` attribute from the appropriate `image_processor`. If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3. Then use \ud83e\udd17 Datasets[`~datasets.Dataset.set_transform`] to apply the transforms on the fly:\n```py\n>>> dataset.set_transform(transforms)\n```\n\n4. Now when you access the image, you'll notice the image processor has added `pixel_values`. You can pass your processed dataset to the model now!\n\n```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: c6b5baf8-a449-4a34-aad5-dc6df4f7905c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"c6b5baf8-a449-4a34-aad5-dc6df4f7905c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f8ccfd99-88e0-470a-becc-16d7ed229a77\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"77dde90ed4160e8c6d60b85c74af9a99898accf63c156343bb79f4fc92bc0584\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"92ddff29-ab1b-4039-a25f-4c48252ba478\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c39f41d58c94f8d1a15fe5885021a22add22dfc84ed9e62378aff7d4f79c0e70\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19240, \"end_char_idx\": 21570, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "If you do not resize images during image augmentation,\nleave this parameter out. By default, `ImageProcessor` will handle the resizing.\n\nIf you wish to normalize images as a part of the augmentation transformation, use the `image_processor.image_mean`,\nand `image_processor.image_std` values.\n</Tip>\n\n3. Then use \ud83e\udd17 Datasets[`~datasets.Dataset.set_transform`] to apply the transforms on the fly:\n```py\n>>> dataset.set_transform(transforms)\n```\n\n4. Now when you access the image, you'll notice the image processor has added `pixel_values`. You can pass your processed dataset to the model now!\n\n```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together.\n\n```py\n>>> def collate_fn(batch):\n...     pixel_values = [item[\"pixel_values\"] for item in batch]\n...     encoding = image_processor.pad(pixel_values, return_tensors=\"pt\")\n...     labels = [item[\"labels\"] for item in batch]\n...     batch = {}\n...     batch[\"pixel_values\"] = encoding[\"pixel_values\"]\n...     batch[\"pixel_mask\"] = encoding[\"pixel_mask\"]\n...     batch[\"labels\"] = labels\n...     return batch\n```\n\n## Multimodal\n\nFor tasks involving multimodal inputs, you'll need a [processor](main_classes/processors) to prepare your dataset for the model. A processor couples together two processing objects such as tokenizer and feature extractor."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 92ddff29-ab1b-4039-a25f-4c48252ba478
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"92ddff29-ab1b-4039-a25f-4c48252ba478\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c6b5baf8-a449-4a34-aad5-dc6df4f7905c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0063149ed445dc0c76f115253cd977d064d5fc2df24b2160ce89d9c107628802\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"aa449462-d49f-4fd2-96d4-3e9f26fabbc3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7196abb711ab229bf55d28e6fc41efa5416af045ca48ec302caa817d181487d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19833, \"end_char_idx\": 22892, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "```py\n>>> dataset[0].keys()\n```\n\nHere is what the image looks like after the transforms are applied. The image has been randomly cropped and it's color properties are different.\n\n```py\n>>> import numpy as np\n>>> import matplotlib.pyplot as plt\n\n>>> img = dataset[0][\"pixel_values\"]\n>>> plt.imshow(img.permute(1, 2, 0))\n```\n\n<div class=\"flex justify-center\">\n    <img src=\"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png\"/>\n</div>\n\n<Tip>\n\nFor tasks like object detection, semantic segmentation, instance segmentation, and panoptic segmentation, `ImageProcessor`\noffers post processing methods. These methods convert model's raw outputs into meaningful predictions such as bounding boxes,\nor segmentation maps.\n\n</Tip>\n\n### Pad\n\nIn some cases, for instance, when fine-tuning [DETR](./model_doc/detr), the model applies scale augmentation at training\ntime. This may cause images to be different sizes in a batch. You can use [`DetrImageProcessor.pad`]\nfrom [`DetrImageProcessor`] and define a custom `collate_fn` to batch images together.\n\n```py\n>>> def collate_fn(batch):\n...     pixel_values = [item[\"pixel_values\"] for item in batch]\n...     encoding = image_processor.pad(pixel_values, return_tensors=\"pt\")\n...     labels = [item[\"labels\"] for item in batch]\n...     batch = {}\n...     batch[\"pixel_values\"] = encoding[\"pixel_values\"]\n...     batch[\"pixel_mask\"] = encoding[\"pixel_mask\"]\n...     batch[\"labels\"] = labels\n...     return batch\n```\n\n## Multimodal\n\nFor tasks involving multimodal inputs, you'll need a [processor](main_classes/processors) to prepare your dataset for the model. A processor couples together two processing objects such as tokenizer and feature extractor.\n\nLoad the [LJ Speech](https://huggingface.co/datasets/lj_speech) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):\n\n```py\n>>> from datasets import load_dataset\n\n>>> lj_speech = load_dataset(\"lj_speech\", split=\"train\")\n```\n\nFor ASR, you're mainly focused on `audio` and `text` so you can remove the other columns:\n\n```py\n>>> lj_speech = lj_speech.map(remove_columns=[\"file\", \"id\", \"normalized_text\"])\n```\n\nNow take a look at the `audio` and `text` columns:\n\n```py\n>>> lj_speech[0][\"audio\"]\n{'array': array([-7.3242188e-04, -7.6293945e-04, -6.4086914e-04, ...,\n         7.3242188e-04,  2.1362305e-04,  6.1035156e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav',\n 'sampling_rate': 22050}\n\n>>> lj_speech[0][\"text\"]\n'Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition'\n```\n\nRemember you should always [resample](preprocessing#audio) your audio dataset's sampling rate to match the sampling rate of the dataset used to pretrain a model!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: aa449462-d49f-4fd2-96d4-3e9f26fabbc3
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/preprocessing", "title": "Preprocess", "tokens": 7343, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"aa449462-d49f-4fd2-96d4-3e9f26fabbc3\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a701d0ef-c3ad-4dc0-92d7-7a8fd611db21\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"eb5f432185a3f46a2f66cde236a8c347570bdb381055e64512097ce307099030\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"92ddff29-ab1b-4039-a25f-4c48252ba478\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/preprocessing\", \"title\": \"Preprocess\", \"tokens\": 7343, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e286f602d198cd0c06b5064766d12afca550545a8737abdf2be565c382f6eb20\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21572, \"end_char_idx\": 23806, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "ref_doc_id": "a701d0ef-c3ad-4dc0-92d7-7a8fd611db21", "chroma:document": "Load the [LJ Speech](https://huggingface.co/datasets/lj_speech) dataset (see the \ud83e\udd17 [Datasets tutorial](https://huggingface.co/docs/datasets/load_hub) for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):\n\n```py\n>>> from datasets import load_dataset\n\n>>> lj_speech = load_dataset(\"lj_speech\", split=\"train\")\n```\n\nFor ASR, you're mainly focused on `audio` and `text` so you can remove the other columns:\n\n```py\n>>> lj_speech = lj_speech.map(remove_columns=[\"file\", \"id\", \"normalized_text\"])\n```\n\nNow take a look at the `audio` and `text` columns:\n\n```py\n>>> lj_speech[0][\"audio\"]\n{'array': array([-7.3242188e-04, -7.6293945e-04, -6.4086914e-04, ...,\n         7.3242188e-04,  2.1362305e-04,  6.1035156e-05], dtype=float32),\n 'path': '/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav',\n 'sampling_rate': 22050}\n\n>>> lj_speech[0][\"text\"]\n'Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition'\n```\n\nRemember you should always [resample](preprocessing#audio) your audio dataset's sampling rate to match the sampling rate of the dataset used to pretrain a model!\n\n```py\n>>> lj_speech = lj_speech.cast_column(\"audio\", Audio(sampling_rate=16_000))\n```\n\nLoad a processor with [`AutoProcessor.from_pretrained`]:\n\n```py\n>>> from transformers import AutoProcessor\n\n>>> processor = AutoProcessor.from_pretrained(\"facebook/wav2vec2-base-960h\")\n```\n\n1. Create a function to process the audio data contained in `array` to `input_values`, and tokenize `text` to `labels`. These are the inputs to the model:\n\n```py\n>>> def prepare_dataset(example):\n...     audio = example[\"audio\"]\n\n...     example.update(processor(audio=audio[\"array\"], text=example[\"text\"], sampling_rate=16000))\n\n...     return example\n```\n\n2. Apply the `prepare_dataset` function to a sample:\n\n```py\n>>> prepare_dataset(lj_speech[0])\n```\n\nThe processor has now added `input_values` and `labels`, and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 02b3524e-2f5f-4da2-b83e-52ca616b3cee
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/sagemaker", "title": "Run training on Amazon SageMaker", "tokens": 278, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"02b3524e-2f5f-4da2-b83e-52ca616b3cee\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/sagemaker\", \"title\": \"Run training on Amazon SageMaker\", \"tokens\": 278, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"a94bb6a3-321e-493e-a3e6-17e24464c59a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/sagemaker\", \"title\": \"Run training on Amazon SageMaker\", \"tokens\": 278, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"7698b2ff621f3e0a64efa530a64dc41f68dc6ea4f228f13ba9b6e5f358029a6b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1220, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "a94bb6a3-321e-493e-a3e6-17e24464c59a", "doc_id": "a94bb6a3-321e-493e-a3e6-17e24464c59a", "ref_doc_id": "a94bb6a3-321e-493e-a3e6-17e24464c59a", "chroma:document": "<!---\nCopyright 2020 The HuggingFace Team. All rights reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\n\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n\n-->\n\n# Run training on Amazon SageMaker\n\nThe documentation has been moved to [hf.co/docs/sagemaker](https://huggingface.co/docs/sagemaker). This page will be removed in `transformers` 5.0. \n\n### Table of Content\n\n- [Train Hugging Face models on Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/train)\n- [Deploy Hugging Face models to Amazon SageMaker with the SageMaker Python SDK](https://huggingface.co/docs/sagemaker/inference)"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 23793a67-2c17-4742-9729-a67ad0ea7dd5
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2342, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"23793a67-2c17-4742-9729-a67ad0ea7dd5\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1397b358-a3fa-490b-bf94-887281c78c33\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0299cc378101295900500be329593638b6156bdf475ecd9bcd1c1eb97ad01c22\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"020ed8be-9509-498f-ae43-05d9a7ab89ff\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"43e1063616f77104376099d65cd307b5b70613459379e078f897eb9995c84cb2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3390, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1397b358-a3fa-490b-bf94-887281c78c33", "doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "ref_doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "chroma:document": "# Share a model\n\nThe last two tutorials showed how you can fine-tune a model with PyTorch, Keras, and \ud83e\udd17 Accelerate for distributed setups. The next step is to share your model with the community! At Hugging Face, we believe in openly sharing knowledge and resources to democratize artificial intelligence for everyone. We encourage you to consider sharing your model with the community to help others save time and resources.\n\nIn this tutorial, you will learn two methods for sharing a trained or fine-tuned model on the [Model Hub](https://huggingface.co/models):\n\n- Programmatically push your files to the Hub.\n- Drag-and-drop your files to the Hub with the web interface.\n\n<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/XvSGPZFEjDY\" title=\"YouTube video player\"\nframeborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope;\npicture-in-picture\" allowfullscreen></iframe>\n\n<Tip>\n\nTo share a model with the community, you need an account on [huggingface.co](https://huggingface.co/join). You can also join an existing organization or create a new one.\n\n</Tip>\n\n## Repository features\n\nEach repository on the Model Hub behaves like a typical GitHub repository. Our repositories offer versioning, commit history, and the ability to visualize differences.\n\nThe Model Hub's built-in versioning is based on git and [git-lfs](https://git-lfs.github.com/). In other words, you can treat one model as one repository, enabling greater access control and scalability. Version control allows *revisions*, a method for pinning a specific version of a model with a commit hash, tag or branch.\n\nAs a result, you can load a specific model version with the `revision` parameter:\n\n```py\n>>> model = AutoModel.from_pretrained(\n...     \"julien-c/EsperBERTo-small\", revision=\"v2.0.1\"  # tag name, or branch name, or commit hash\n... )\n```\n\nFiles are also easily edited in a repository, and you can view the commit history as well as the difference:\n\n![vis_diff](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vis_diff.png)\n\n## Setup\n\nBefore sharing a model to the Hub, you will need your Hugging Face credentials. If you have access to a terminal, run the following command in the virtual environment where \ud83e\udd17 Transformers is installed. This will store your access token in your Hugging Face cache folder (`~/.cache/` by default):\n\n```bash\nhuggingface-cli login\n```\n\nIf you are using a notebook like Jupyter or Colaboratory, make sure you have the [`huggingface_hub`](https://huggingface.co/docs/hub/adding-a-library) library installed. This library allows you to programmatically interact with the Hub.\n\n```bash\npip install huggingface_hub\n```\n\nThen use `notebook_login` to sign-in to the Hub, and follow the link [here](https://huggingface.co/settings/token) to generate a token to login with:\n\n```py\n>>> from huggingface_hub import notebook_login\n\n>>> notebook_login()\n```\n\n## Convert a model for all frameworks\n\nTo ensure your model can be used by someone working with a different framework, we recommend you convert and upload your model with both PyTorch and TensorFlow checkpoints. While users are still able to load your model from a different framework if you skip this step, it will be slower because \ud83e\udd17 Transformers will need to convert the checkpoint on-the-fly.\n\nConverting a checkpoint for another framework is easy."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 020ed8be-9509-498f-ae43-05d9a7ab89ff
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2342, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"020ed8be-9509-498f-ae43-05d9a7ab89ff\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1397b358-a3fa-490b-bf94-887281c78c33\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0299cc378101295900500be329593638b6156bdf475ecd9bcd1c1eb97ad01c22\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"23793a67-2c17-4742-9729-a67ad0ea7dd5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cb8b04670ba0a27daae93b6d03eff3e15e71e93938397e4cfe165828264a66a6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e8e1eaac-53e4-4a86-bf50-53207bb9f004\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"94d6c6f43134da7416495ca6df382a5e47c027b53fab7c87ac6ca5b87eacead1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1982, \"end_char_idx\": 4909, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1397b358-a3fa-490b-bf94-887281c78c33", "doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "ref_doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "chroma:document": "[vis_diff](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vis_diff.png)\n\n## Setup\n\nBefore sharing a model to the Hub, you will need your Hugging Face credentials. If you have access to a terminal, run the following command in the virtual environment where \ud83e\udd17 Transformers is installed. This will store your access token in your Hugging Face cache folder (`~/.cache/` by default):\n\n```bash\nhuggingface-cli login\n```\n\nIf you are using a notebook like Jupyter or Colaboratory, make sure you have the [`huggingface_hub`](https://huggingface.co/docs/hub/adding-a-library) library installed. This library allows you to programmatically interact with the Hub.\n\n```bash\npip install huggingface_hub\n```\n\nThen use `notebook_login` to sign-in to the Hub, and follow the link [here](https://huggingface.co/settings/token) to generate a token to login with:\n\n```py\n>>> from huggingface_hub import notebook_login\n\n>>> notebook_login()\n```\n\n## Convert a model for all frameworks\n\nTo ensure your model can be used by someone working with a different framework, we recommend you convert and upload your model with both PyTorch and TensorFlow checkpoints. While users are still able to load your model from a different framework if you skip this step, it will be slower because \ud83e\udd17 Transformers will need to convert the checkpoint on-the-fly.\n\nConverting a checkpoint for another framework is easy. Make sure you have PyTorch and TensorFlow installed (see [here](installation) for installation instructions), and then find the specific model for your task in the other framework. \n\n<frameworkcontent>\n<pt>\nSpecify `from_tf=True` to convert a checkpoint from TensorFlow to PyTorch:\n\n```py\n>>> pt_model = DistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_tf=True)\n>>> pt_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</pt>\n<tf>\nSpecify `from_pt=True` to convert a checkpoint from PyTorch to TensorFlow:\n\n```py\n>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_pt=True)\n```\n\nThen you can save your new TensorFlow model with its new checkpoint:\n\n```py\n>>> tf_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</tf>\n<jax>\nIf a model is available in Flax, you can also convert a checkpoint from PyTorch to Flax:\n\n```py\n>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(\n...     \"path/to/awesome-name-you-picked\", from_pt=True\n... )\n```\n</jax>\n</frameworkcontent>\n\n## Push a model during training\n\n<frameworkcontent>\n<pt>\n<Youtube id=\"Z1-XMy-GNLQ\"/>\n\nSharing a model to the Hub is as simple as adding an extra parameter or callback. Remember from the [fine-tuning tutorial](training), the [`TrainingArguments`] class is where you specify hyperparameters and additional training options. One of these training options includes the ability to push a model directly to the Hub."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: e8e1eaac-53e4-4a86-bf50-53207bb9f004
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2342, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"e8e1eaac-53e4-4a86-bf50-53207bb9f004\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1397b358-a3fa-490b-bf94-887281c78c33\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0299cc378101295900500be329593638b6156bdf475ecd9bcd1c1eb97ad01c22\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"020ed8be-9509-498f-ae43-05d9a7ab89ff\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"85a56a74c660ae60959ce13939b5d61cfcd98e61ac0d753a0b5a3a89651cdba1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"667fd23f-8d32-44bb-94bc-d713f9abe3d4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"359803c6316cafb665e7d0c7efae85dc176287329b68d5a91cf8411c971d0036\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3336, \"end_char_idx\": 6480, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1397b358-a3fa-490b-bf94-887281c78c33", "doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "ref_doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "chroma:document": "Converting a checkpoint for another framework is easy. Make sure you have PyTorch and TensorFlow installed (see [here](installation) for installation instructions), and then find the specific model for your task in the other framework. \n\n<frameworkcontent>\n<pt>\nSpecify `from_tf=True` to convert a checkpoint from TensorFlow to PyTorch:\n\n```py\n>>> pt_model = DistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_tf=True)\n>>> pt_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</pt>\n<tf>\nSpecify `from_pt=True` to convert a checkpoint from PyTorch to TensorFlow:\n\n```py\n>>> tf_model = TFDistilBertForSequenceClassification.from_pretrained(\"path/to/awesome-name-you-picked\", from_pt=True)\n```\n\nThen you can save your new TensorFlow model with its new checkpoint:\n\n```py\n>>> tf_model.save_pretrained(\"path/to/awesome-name-you-picked\")\n```\n</tf>\n<jax>\nIf a model is available in Flax, you can also convert a checkpoint from PyTorch to Flax:\n\n```py\n>>> flax_model = FlaxDistilBertForSequenceClassification.from_pretrained(\n...     \"path/to/awesome-name-you-picked\", from_pt=True\n... )\n```\n</jax>\n</frameworkcontent>\n\n## Push a model during training\n\n<frameworkcontent>\n<pt>\n<Youtube id=\"Z1-XMy-GNLQ\"/>\n\nSharing a model to the Hub is as simple as adding an extra parameter or callback. Remember from the [fine-tuning tutorial](training), the [`TrainingArguments`] class is where you specify hyperparameters and additional training options. One of these training options includes the ability to push a model directly to the Hub. Set `push_to_hub=True` in your [`TrainingArguments`]:\n\n```py\n>>> training_args = TrainingArguments(output_dir=\"my-awesome-model\", push_to_hub=True)\n```\n\nPass your training arguments as usual to [`Trainer`]:\n\n```py\n>>> trainer = Trainer(\n...     model=model,\n...     args=training_args,\n...     train_dataset=small_train_dataset,\n...     eval_dataset=small_eval_dataset,\n...     compute_metrics=compute_metrics,\n... )\n```\n\nAfter you fine-tune your model, call [`~transformers.Trainer.push_to_hub`] on [`Trainer`] to push the trained model to the Hub. \ud83e\udd17 Transformers will even automatically add training hyperparameters, training results and framework versions to your model card!\n\n```py\n>>> trainer.push_to_hub()\n```\n</pt>\n<tf>\nShare a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\n\n- An output directory for your model.\n- A tokenizer.\n- The `hub_model_id`, which is your Hub username and model name.\n\n```py\n>>> from transformers import PushToHubCallback\n\n>>> push_to_hub_callback = PushToHubCallback(\n...     output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\n... )\n```\n\nAdd the callback to [`fit`](https://keras.io/api/models/model_training_apis/), and \ud83e\udd17 Transformers will push the trained model to the Hub:\n\n```py\n>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)\n```\n</tf>\n</frameworkcontent>\n\n## Use the `push_to_hub` function\n\nYou can also call `push_to_hub` directly on your model to upload it to the Hub."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 667fd23f-8d32-44bb-94bc-d713f9abe3d4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2342, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"667fd23f-8d32-44bb-94bc-d713f9abe3d4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1397b358-a3fa-490b-bf94-887281c78c33\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0299cc378101295900500be329593638b6156bdf475ecd9bcd1c1eb97ad01c22\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e8e1eaac-53e4-4a86-bf50-53207bb9f004\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c538258ffa9d2ffd331f21d33daa02dc233cf64c16d5272fa82d156b65284a65\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"512c4727-a218-4ba2-b6d7-32b6e2282772\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e79843da10f7db55e0b89d3b645885e719cdc4703c2e0c9759e2c7a14ae32657\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4822, \"end_char_idx\": 7966, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1397b358-a3fa-490b-bf94-887281c78c33", "doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "ref_doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "chroma:document": "One of these training options includes the ability to push a model directly to the Hub. Set `push_to_hub=True` in your [`TrainingArguments`]:\n\n```py\n>>> training_args = TrainingArguments(output_dir=\"my-awesome-model\", push_to_hub=True)\n```\n\nPass your training arguments as usual to [`Trainer`]:\n\n```py\n>>> trainer = Trainer(\n...     model=model,\n...     args=training_args,\n...     train_dataset=small_train_dataset,\n...     eval_dataset=small_eval_dataset,\n...     compute_metrics=compute_metrics,\n... )\n```\n\nAfter you fine-tune your model, call [`~transformers.Trainer.push_to_hub`] on [`Trainer`] to push the trained model to the Hub. \ud83e\udd17 Transformers will even automatically add training hyperparameters, training results and framework versions to your model card!\n\n```py\n>>> trainer.push_to_hub()\n```\n</pt>\n<tf>\nShare a model to the Hub with [`PushToHubCallback`]. In the [`PushToHubCallback`] function, add:\n\n- An output directory for your model.\n- A tokenizer.\n- The `hub_model_id`, which is your Hub username and model name.\n\n```py\n>>> from transformers import PushToHubCallback\n\n>>> push_to_hub_callback = PushToHubCallback(\n...     output_dir=\"./your_model_save_path\", tokenizer=tokenizer, hub_model_id=\"your-username/my-awesome-model\"\n... )\n```\n\nAdd the callback to [`fit`](https://keras.io/api/models/model_training_apis/), and \ud83e\udd17 Transformers will push the trained model to the Hub:\n\n```py\n>>> model.fit(tf_train_dataset, validation_data=tf_validation_dataset, epochs=3, callbacks=push_to_hub_callback)\n```\n</tf>\n</frameworkcontent>\n\n## Use the `push_to_hub` function\n\nYou can also call `push_to_hub` directly on your model to upload it to the Hub.\n\nSpecify your model name in `push_to_hub`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-model\")\n```\n\nThis creates a repository under your username with the model name `my-awesome-model`. Users can now load your model with the `from_pretrained` function:\n\n```py\n>>> from transformers import AutoModel\n\n>>> model = AutoModel.from_pretrained(\"your_username/my-awesome-model\")\n```\n\nIf you belong to an organization and want to push your model under the organization name instead, just add it to the `repo_id`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-org/my-awesome-model\")\n```\n\nThe `push_to_hub` function can also be used to add other files to a model repository. For example, add a tokenizer to a model repository:\n\n```py\n>>> tokenizer.push_to_hub(\"my-awesome-model\")\n```\n\nOr perhaps you'd like to add the TensorFlow version of your fine-tuned PyTorch model:\n\n```py\n>>> tf_model.push_to_hub(\"my-awesome-model\")\n```\n\nNow when you navigate to your Hugging Face profile, you should see your newly created model repository. Clicking on the **Files** tab will display all the files you've uploaded to the repository.\n\nFor more details on how to create and upload files to a repository, refer to the Hub documentation [here](https://huggingface.co/docs/hub/how-to-upstream).\n\n## Upload with the web interface\n\nUsers who prefer a no-code approach are able to upload a model through the Hub's web interface. Visit [huggingface.co/new](https://huggingface.co/new) to create a new repository:\n\n!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 512c4727-a218-4ba2-b6d7-32b6e2282772
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/model_sharing", "title": "Share a model", "tokens": 2342, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"512c4727-a218-4ba2-b6d7-32b6e2282772\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1397b358-a3fa-490b-bf94-887281c78c33\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0299cc378101295900500be329593638b6156bdf475ecd9bcd1c1eb97ad01c22\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"667fd23f-8d32-44bb-94bc-d713f9abe3d4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/model_sharing\", \"title\": \"Share a model\", \"tokens\": 2342, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9f1ec89db39ef9bd1805738b5f258676ce8d1ede8aacbef36dd9c662ecef55cd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6482, \"end_char_idx\": 9463, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1397b358-a3fa-490b-bf94-887281c78c33", "doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "ref_doc_id": "1397b358-a3fa-490b-bf94-887281c78c33", "chroma:document": "Specify your model name in `push_to_hub`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-model\")\n```\n\nThis creates a repository under your username with the model name `my-awesome-model`. Users can now load your model with the `from_pretrained` function:\n\n```py\n>>> from transformers import AutoModel\n\n>>> model = AutoModel.from_pretrained(\"your_username/my-awesome-model\")\n```\n\nIf you belong to an organization and want to push your model under the organization name instead, just add it to the `repo_id`:\n\n```py\n>>> pt_model.push_to_hub(\"my-awesome-org/my-awesome-model\")\n```\n\nThe `push_to_hub` function can also be used to add other files to a model repository. For example, add a tokenizer to a model repository:\n\n```py\n>>> tokenizer.push_to_hub(\"my-awesome-model\")\n```\n\nOr perhaps you'd like to add the TensorFlow version of your fine-tuned PyTorch model:\n\n```py\n>>> tf_model.push_to_hub(\"my-awesome-model\")\n```\n\nNow when you navigate to your Hugging Face profile, you should see your newly created model repository. Clicking on the **Files** tab will display all the files you've uploaded to the repository.\n\nFor more details on how to create and upload files to a repository, refer to the Hub documentation [here](https://huggingface.co/docs/hub/how-to-upstream).\n\n## Upload with the web interface\n\nUsers who prefer a no-code approach are able to upload a model through the Hub's web interface. Visit [huggingface.co/new](https://huggingface.co/new) to create a new repository:\n\n![new_model_repo](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/new_model_repo.png)\n\nFrom here, add some information about your model:\n\n- Select the **owner** of the repository. This can be yourself or any of the organizations you belong to.\n- Pick a name for your model, which will also be the repository name.\n- Choose whether your model is public or private.\n- Specify the license usage for your model.\n\nNow click on the **Files** tab and click on the **Add file** button to upload a new file to your repository. Then drag-and-drop a file to upload and add a commit message.\n\n![upload_file](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/upload_file.png)\n\n## Add a model card\n\nTo make sure users understand your model's capabilities, limitations, potential biases and ethical considerations, please add a model card to your repository. The model card is defined in the `README.md` file. You can add a model card by:\n\n* Manually creating and uploading a `README.md` file.\n* Clicking on the **Edit model card** button in your model repository.\n\nTake a look at the DistilBert [model card](https://huggingface.co/distilbert/distilbert-base-uncased) for a good example of the type of information a model card should include. For more details about other options you can control in the `README.md` file such as a model's carbon footprint or widget examples, refer to the documentation [here](https://huggingface.co/docs/hub/models-cards)."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: ddd793c3-21a6-4dba-b5a1-caa10ddf8401
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"ddd793c3-21a6-4dba-b5a1-caa10ddf8401\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"60c57af1-84cd-424c-8b69-fd671ce0ff92\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0d9cb863d56d182bebbd4a4355287f635b8317ff77453822bfade7a18a9ba103\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2914, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "# GPU inference\n\nGPUs are the standard choice of hardware for machine learning, unlike CPUs, because they are optimized for memory bandwidth and parallelism. To keep up with the larger sizes of modern models or to run these large models on existing and older hardware, there are several optimizations you can use to speed up GPU inference. In this guide, you'll learn how to use FlashAttention-2 (a more memory-efficient attention mechanism), BetterTransformer (a PyTorch native fastpath execution), and bitsandbytes to quantize your model to a lower precision. Finally, learn how to use \ud83e\udd17 Optimum to accelerate inference with ONNX Runtime on Nvidia and AMD GPUs.\n\n<Tip>\n\nThe majority of the optimizations described here also apply to multi-GPU setups!\n\n</Tip>\n\n## FlashAttention-2\n\n<Tip>\n\nFlashAttention-2 is experimental and may change considerably in future versions.\n\n</Tip>\n\n[FlashAttention-2](https://huggingface.co/papers/2205.14135) is a faster and more efficient implementation of the standard attention mechanism that can significantly speedup inference by:\n\n1. additionally parallelizing the attention computation over sequence length\n2. partitioning the work between GPU threads to reduce communication and shared memory reads/writes between them\n\nFlashAttention-2 is currently supported for the following architectures:\n* [Bark](https://huggingface.co/docs/transformers/model_doc/bark#transformers.BarkModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DistilBert](https://huggingface.co/docs/transformers/model_doc/distilbert#transformers.DistilBertModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeo](https://huggingface.co/docs/transformers/model_doc/gpt_neo#transformers.GPTNeoModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj#transformers.GPTJModel)\n* [Idefics2](https://huggingface.co/docs/transformers/model_doc/idefics2#transformers.Idefics2Model)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 60c57af1-84cd-424c-8b69-fd671ce0ff92
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"60c57af1-84cd-424c-8b69-fd671ce0ff92\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ddd793c3-21a6-4dba-b5a1-caa10ddf8401\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c466680d57251f4e44d6a295f6a0eb06cddc08583f7b083aa31c60919ef3812f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2f0a61e8-69bb-4d76-a2e3-3b7edcff04d4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6f91445addef4dceb9b39a8cb42d9e5e9d49248b81eb8b30b5c57fa68a2e9aea\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1730, \"end_char_idx\": 4110, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DistilBert](https://huggingface.co/docs/transformers/model_doc/distilbert#transformers.DistilBertModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeo](https://huggingface.co/docs/transformers/model_doc/gpt_neo#transformers.GPTNeoModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [GPT-J](https://huggingface.co/docs/transformers/model_doc/gptj#transformers.GPTJModel)\n* [Idefics2](https://huggingface.co/docs/transformers/model_doc/idefics2#transformers.Idefics2Model)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [Llava](https://huggingface.co/docs/transformers/model_doc/llava)\n* [Llava-NeXT](https://huggingface.co/docs/transformers/model_doc/llava_next)\n* [Llava-NeXT-Video](https://huggingface.co/docs/transformers/model_doc/llava_next_video)\n* [VipLlava](https://huggingface.co/docs/transformers/model_doc/vipllava)\n* [VideoLlava](https://huggingface.co/docs/transformers/model_doc/video_llava)\n* [M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)\n* [MBart](https://huggingface.co/docs/transformers/model_doc/mbart#transformers.MBartModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [NLLB](https://huggingface."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2f0a61e8-69bb-4d76-a2e3-3b7edcff04d4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2f0a61e8-69bb-4d76-a2e3-3b7edcff04d4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"60c57af1-84cd-424c-8b69-fd671ce0ff92\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"155a45bd860a15db018f888f5bf51804b9593e7e0592a486e2b9a1ea6e153a24\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5858f365-a9a1-463f-8580-2244e4ab73dc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"993283a283ced790472473ab1f5121c4d0de427032e84e77ba4e1b65bb6fc005\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2863, \"end_char_idx\": 5250, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [Llava](https://huggingface.co/docs/transformers/model_doc/llava)\n* [Llava-NeXT](https://huggingface.co/docs/transformers/model_doc/llava_next)\n* [Llava-NeXT-Video](https://huggingface.co/docs/transformers/model_doc/llava_next_video)\n* [VipLlava](https://huggingface.co/docs/transformers/model_doc/vipllava)\n* [VideoLlava](https://huggingface.co/docs/transformers/model_doc/video_llava)\n* [M2M100](https://huggingface.co/docs/transformers/model_doc/m2m_100)\n* [MBart](https://huggingface.co/docs/transformers/model_doc/mbart#transformers.MBartModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [OPT](https://huggingface.co/docs/transformers/model_doc/opt#transformers.OPTModel)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 5858f365-a9a1-463f-8580-2244e4ab73dc
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5858f365-a9a1-463f-8580-2244e4ab73dc\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2f0a61e8-69bb-4d76-a2e3-3b7edcff04d4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"4ad5517a01f385e117f9e610ccfdf6ca8b359f4d9ae7eac9f5db223b11fe0a94\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4eb8b208-c2f3-40b2-b387-43d814981b38\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7e75b03d3be713803c6962606a42781dc22ee6fc359f9428072ad35860d0aafd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4060, \"end_char_idx\": 6586, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "MusicgenMelodyModel)\n* [NLLB](https://huggingface.co/docs/transformers/model_doc/nllb)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [OPT](https://huggingface.co/docs/transformers/model_doc/opt#transformers.OPTModel)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Phi3](https://huggingface.co/docs/transformers/model_doc/phi3#transformers.Phi3Model)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Wav2Vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n\nYou can request to add FlashAttention-2 support for another model by opening a GitHub Issue or Pull Request.\n\nBefore you begin, make sure you have FlashAttention-2 installed.\n\n<hfoptions id=\"install\">\n<hfoption id=\"NVIDIA\">\n\n```bash\npip install flash-attn --no-build-isolation\n```\n\nWe strongly suggest referring to the detailed [installation instructions](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features) to learn more about supported hardware and data types!\n\n</hfoption>\n<hfoption id=\"AMD\">\n\nFlashAttention-2 is also supported on AMD GPUs and current support is limited to **Instinct MI210**, **Instinct MI250** and **Instinct MI300**. We strongly suggest using this [Dockerfile](https://github.com/huggingface/optimum-amd/tree/main/docker/transformers-pytorch-amd-gpu-flash/Dockerfile) to use FlashAttention-2 on AMD GPUs."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 4eb8b208-c2f3-40b2-b387-43d814981b38
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4eb8b208-c2f3-40b2-b387-43d814981b38\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5858f365-a9a1-463f-8580-2244e4ab73dc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f1306f9f51ac19cdf63f8a1782f1c9166ebe3838d05635efafb6cedfca1dfc2d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"73beca23-85bd-4c9b-927e-2b4683a377c0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2b03646c76fb23028f0bcde26f2ad2e40b1ad2163ee043c48c554fa4d3ae26a0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5153, \"end_char_idx\": 7565, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n\nYou can request to add FlashAttention-2 support for another model by opening a GitHub Issue or Pull Request.\n\nBefore you begin, make sure you have FlashAttention-2 installed.\n\n<hfoptions id=\"install\">\n<hfoption id=\"NVIDIA\">\n\n```bash\npip install flash-attn --no-build-isolation\n```\n\nWe strongly suggest referring to the detailed [installation instructions](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#installation-and-features) to learn more about supported hardware and data types!\n\n</hfoption>\n<hfoption id=\"AMD\">\n\nFlashAttention-2 is also supported on AMD GPUs and current support is limited to **Instinct MI210**, **Instinct MI250** and **Instinct MI300**. We strongly suggest using this [Dockerfile](https://github.com/huggingface/optimum-amd/tree/main/docker/transformers-pytorch-amd-gpu-flash/Dockerfile) to use FlashAttention-2 on AMD GPUs.\n\n</hfoption>\n</hfoptions>\n\nTo enable FlashAttention-2, pass the argument `attn_implementation=\"flash_attention_2\"` to [`~AutoModelForCausalLM.from_pretrained`]:\n\n```python\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    torch_dtype=torch.bfloat16,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n<Tip>\n\nFlashAttention-2 can only be used when the model's dtype is `fp16` or `bf16`. Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 73beca23-85bd-4c9b-927e-2b4683a377c0
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"73beca23-85bd-4c9b-927e-2b4683a377c0\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4eb8b208-c2f3-40b2-b387-43d814981b38\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"3836a6e149d30ea2ddd0655d29bad86f9782b8d22a86c909fb5105ad448184cb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e9bbf1f7-8c77-4464-862e-e303d5e54449\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3329164b29608d66e7683702b3159761ea523b1a556afb98036610bf6198e2b4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6222, \"end_char_idx\": 8849, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "</hfoption>\n<hfoption id=\"AMD\">\n\nFlashAttention-2 is also supported on AMD GPUs and current support is limited to **Instinct MI210**, **Instinct MI250** and **Instinct MI300**. We strongly suggest using this [Dockerfile](https://github.com/huggingface/optimum-amd/tree/main/docker/transformers-pytorch-amd-gpu-flash/Dockerfile) to use FlashAttention-2 on AMD GPUs.\n\n</hfoption>\n</hfoptions>\n\nTo enable FlashAttention-2, pass the argument `attn_implementation=\"flash_attention_2\"` to [`~AutoModelForCausalLM.from_pretrained`]:\n\n```python\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    torch_dtype=torch.bfloat16,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n<Tip>\n\nFlashAttention-2 can only be used when the model's dtype is `fp16` or `bf16`. Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference. For example, you can combine FlashAttention-2 with 8-bit or 4-bit quantization:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\n# load in 8bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_8bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n\n# load in 4bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n### Expected speedups\n\nYou can benefit from considerable speedups for inference, especially for inputs with long sequences. However, since FlashAttention-2 does not support computing attention scores with padding tokens, you must manually pad/unpad the attention scores for batched inference when the sequence contains padding tokens. This leads to a significant slowdown for batched generations with padding tokens.\n\nTo overcome this, you should use FlashAttention-2 without padding tokens in the sequence during training (by packing a dataset or [concatenating sequences](https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py#L516) until reaching the maximum sequence length)."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: e9bbf1f7-8c77-4464-862e-e303d5e54449
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"e9bbf1f7-8c77-4464-862e-e303d5e54449\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"73beca23-85bd-4c9b-927e-2b4683a377c0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"4bf341f1609b54bc4b94cfbc108cf34d56973edad64ac7f64c907d6deda4dd3a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3bf2c3cc-bba8-4062-884d-87dffcf07550\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"021412bd0dd7de709897a2acfcfb1a710c3907684b4992499ef52384ec4955f5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7164, \"end_char_idx\": 10468, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "Make sure to cast your model to the appropriate dtype and load them on a supported device before using FlashAttention-2.\n\n<br>\n\nYou can also set `use_flash_attention_2=True` to enable FlashAttention-2 but it is deprecated in favor of `attn_implementation=\"flash_attention_2\"`.\n\n</Tip>\n\nFlashAttention-2 can be combined with other optimization techniques like quantization to further speedup inference. For example, you can combine FlashAttention-2 with 8-bit or 4-bit quantization:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, LlamaForCausalLM\n\nmodel_id = \"tiiuae/falcon-7b\"\ntokenizer = AutoTokenizer.from_pretrained(model_id)\n\n# load in 8bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_8bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n\n# load in 4bit\nmodel = AutoModelForCausalLM.from_pretrained(\n    model_id,\n    load_in_4bit=True,\n    attn_implementation=\"flash_attention_2\",\n)\n```\n\n### Expected speedups\n\nYou can benefit from considerable speedups for inference, especially for inputs with long sequences. However, since FlashAttention-2 does not support computing attention scores with padding tokens, you must manually pad/unpad the attention scores for batched inference when the sequence contains padding tokens. This leads to a significant slowdown for batched generations with padding tokens.\n\nTo overcome this, you should use FlashAttention-2 without padding tokens in the sequence during training (by packing a dataset or [concatenating sequences](https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py#L516) until reaching the maximum sequence length).\n\nFor a single forward pass on [tiiuae/falcon-7b](https://hf.co/tiiuae/falcon-7b) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/falcon-7b-inference-large-seqlen.png\">\n</div>\n\nFor a single forward pass on [meta-llama/Llama-7b-hf](https://hf.co/meta-llama/Llama-7b-hf) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-7b-inference-large-seqlen.png\">\n</div>\n\nFor sequences with padding tokens (generating with padding tokens), you need to unpad/pad the input sequences to correctly compute the attention scores. With a relatively small sequence length, a single forward pass creates overhead leading to a small speedup (in the example below, 30% of the input is filled with padding tokens):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-small-seqlen-padding.png\">\n</div>\n\nBut for larger sequence lengths, you can expect even more speedup benefits:\n\n<Tip>\n\nFlashAttention is more memory efficient, meaning you can train on much larger sequence lengths without running into out-of-memory issues. You can potentially reduce memory usage up to 20x for larger sequence lengths. Take a look at the [flash-attention](https://github.com/Dao-AILab/flash-attention) repository for more details."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 3bf2c3cc-bba8-4062-884d-87dffcf07550
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3bf2c3cc-bba8-4062-884d-87dffcf07550\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e9bbf1f7-8c77-4464-862e-e303d5e54449\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1f0809105211ba1a6c33bb20c317778e5213b91040d244e9ca27298ca113394e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"92ad7628-bd17-4c40-9dab-0bfd6f8b1121\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"64b216d09d783aae60e7b812a523d8ea2808fcd2cf6a3bbeae6c6e3c4cea44e1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8851, \"end_char_idx\": 11951, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "For a single forward pass on [tiiuae/falcon-7b](https://hf.co/tiiuae/falcon-7b) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/falcon-7b-inference-large-seqlen.png\">\n</div>\n\nFor a single forward pass on [meta-llama/Llama-7b-hf](https://hf.co/meta-llama/Llama-7b-hf) with a sequence length of 4096 and various batch sizes without padding tokens, the expected speedup is:\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-7b-inference-large-seqlen.png\">\n</div>\n\nFor sequences with padding tokens (generating with padding tokens), you need to unpad/pad the input sequences to correctly compute the attention scores. With a relatively small sequence length, a single forward pass creates overhead leading to a small speedup (in the example below, 30% of the input is filled with padding tokens):\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-small-seqlen-padding.png\">\n</div>\n\nBut for larger sequence lengths, you can expect even more speedup benefits:\n\n<Tip>\n\nFlashAttention is more memory efficient, meaning you can train on much larger sequence lengths without running into out-of-memory issues. You can potentially reduce memory usage up to 20x for larger sequence lengths. Take a look at the [flash-attention](https://github.com/Dao-AILab/flash-attention) repository for more details.\n\n</Tip>\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-large-seqlen-padding.png\">\n</div>\n\n## PyTorch scaled dot product attention\n\nPyTorch's [`torch.nn.functional.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA) can also call FlashAttention and memory-efficient attention kernels under the hood. SDPA support is currently being added natively in Transformers and is used by default for `torch>=2.1.1` when an implementation is available. You may also set `attn_implementation=\"sdpa\"` in `from_pretrained()` to explicitly request SDPA to be used.\n\nFor now, Transformers supports SDPA inference and training for the following architectures:\n* [Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer#transformers.ASTModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Bert](https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 92ad7628-bd17-4c40-9dab-0bfd6f8b1121
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"92ad7628-bd17-4c40-9dab-0bfd6f8b1121\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3bf2c3cc-bba8-4062-884d-87dffcf07550\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1e4a0dbb541513f747978a71e580a939feff9b95a92d62f0459aa1cd254be91f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"110874aa-1040-4069-9ad5-1f4cf70a0e4c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"12d08406588eebe476190007984ea2af1f71d1f42262e6fb3473c5ba372023cd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10470, \"end_char_idx\": 13149, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "</Tip>\n\n<div style=\"text-align: center\">\n<img src=\"https://huggingface.co/datasets/ybelkada/documentation-images/resolve/main/llama-2-large-seqlen-padding.png\">\n</div>\n\n## PyTorch scaled dot product attention\n\nPyTorch's [`torch.nn.functional.scaled_dot_product_attention`](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention.html) (SDPA) can also call FlashAttention and memory-efficient attention kernels under the hood. SDPA support is currently being added natively in Transformers and is used by default for `torch>=2.1.1` when an implementation is available. You may also set `attn_implementation=\"sdpa\"` in `from_pretrained()` to explicitly request SDPA to be used.\n\nFor now, Transformers supports SDPA inference and training for the following architectures:\n* [Audio Spectrogram Transformer](https://huggingface.co/docs/transformers/model_doc/audio-spectrogram-transformer#transformers.ASTModel)\n* [Bart](https://huggingface.co/docs/transformers/model_doc/bart#transformers.BartModel)\n* [Bert](https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel)\n* [Chameleon](https://huggingface.co/docs/transformers/model_doc/chameleon#transformers.Chameleon)\n* [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPModel)\n* [Cohere](https://huggingface.co/docs/transformers/model_doc/cohere#transformers.CohereModel)\n* [Dbrx](https://huggingface.co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DeiT](https://huggingface.co/docs/transformers/model_doc/deit#transformers.DeiTModel)\n* [Dpr](https://huggingface.co/docs/transformers/model_doc/dpr#transformers.DprReader)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [PaliGemma](https://huggingface.co/docs/transformers/model_doc/paligemma#transformers."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 110874aa-1040-4069-9ad5-1f4cf70a0e4c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"110874aa-1040-4069-9ad5-1f4cf70a0e4c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"92ad7628-bd17-4c40-9dab-0bfd6f8b1121\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"678818867192c70fdacedeb1596bf13deddd4a2d5dcc450bc49ecb61a691820c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dc0829ba-b49e-4029-9cb2-e3feffc11f08\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3dc8ebd763754934535f3e9dc3f0957923c741fecec9525275569eb91eb005a1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11902, \"end_char_idx\": 14362, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "co/docs/transformers/model_doc/dbrx#transformers.DbrxModel)\n* [DeiT](https://huggingface.co/docs/transformers/model_doc/deit#transformers.DeiTModel)\n* [Dpr](https://huggingface.co/docs/transformers/model_doc/dpr#transformers.DprReader)\n* [Falcon](https://huggingface.co/docs/transformers/model_doc/falcon#transformers.FalconModel)\n* [Gemma](https://huggingface.co/docs/transformers/model_doc/gemma#transformers.GemmaModel)\n* [Gemma2](https://huggingface.co/docs/transformers/model_doc/gemma2#transformers.Gemma2Model)\n* [GPT2](https://huggingface.co/docs/transformers/model_doc/gpt2)\n* [GPTBigCode](https://huggingface.co/docs/transformers/model_doc/gpt_bigcode#transformers.GPTBigCodeModel)\n* [GPTNeoX](https://huggingface.co/docs/transformers/model_doc/gpt_neox#transformers.GPTNeoXModel)\n* [JetMoe](https://huggingface.co/docs/transformers/model_doc/jetmoe#transformers.JetMoeModel)\n* [Jamba](https://huggingface.co/docs/transformers/model_doc/jamba#transformers.JambaModel)\n* [Llama](https://huggingface.co/docs/transformers/model_doc/llama#transformers.LlamaModel)\n* [OLMo](https://huggingface.co/docs/transformers/model_doc/olmo#transformers.OlmoModel)\n* [PaliGemma](https://huggingface.co/docs/transformers/model_doc/paligemma#transformers.PaliGemmaForConditionalGeneration)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Idefics](https://huggingface.co/docs/transformers/model_doc/idefics#transformers.IdeficsModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [ViT](https://huggingface.co/docs/transformers/model_doc/vit#transformers."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: dc0829ba-b49e-4029-9cb2-e3feffc11f08
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"dc0829ba-b49e-4029-9cb2-e3feffc11f08\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"110874aa-1040-4069-9ad5-1f4cf70a0e4c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"310a05822310ca57c21a990275192083411dbf1f0c17d9d014d52d0ec5caf648\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2ac61883-fb41-444a-b348-51ef0b96f637\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"04fb98d13661d7cc2e679890c3af30f0e6c52b1bd95e5853b3fd508296a349b1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13095, \"end_char_idx\": 15522, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "co/docs/transformers/model_doc/paligemma#transformers.PaliGemmaForConditionalGeneration)\n* [Phi](https://huggingface.co/docs/transformers/model_doc/phi#transformers.PhiModel)\n* [Idefics](https://huggingface.co/docs/transformers/model_doc/idefics#transformers.IdeficsModel)\n* [Whisper](https://huggingface.co/docs/transformers/model_doc/whisper#transformers.WhisperModel)\n* [Mistral](https://huggingface.co/docs/transformers/model_doc/mistral#transformers.MistralModel)\n* [Mixtral](https://huggingface.co/docs/transformers/model_doc/mixtral#transformers.MixtralModel)\n* [StableLm](https://huggingface.co/docs/transformers/model_doc/stablelm#transformers.StableLmModel)\n* [Starcoder2](https://huggingface.co/docs/transformers/model_doc/starcoder2#transformers.Starcoder2Model)\n* [Qwen2](https://huggingface.co/docs/transformers/model_doc/qwen2#transformers.Qwen2Model)\n* [Qwen2MoE](https://huggingface.co/docs/transformers/model_doc/qwen2_moe#transformers.Qwen2MoeModel)\n* [Musicgen](https://huggingface.co/docs/transformers/model_doc/musicgen#transformers.MusicgenModel)\n* [MusicGen Melody](https://huggingface.co/docs/transformers/model_doc/musicgen_melody#transformers.MusicgenMelodyModel)\n* [ViT](https://huggingface.co/docs/transformers/model_doc/vit#transformers.ViTModel)\n* [ViTHybrid](https://huggingface.co/docs/transformers/model_doc/vit_hybrid#transformers.ViTHybridModel)\n* [ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae#transformers.ViTMAEModel)\n* [ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn#transformers.ViTMSNModel)\n* [VideoMAE](https://huggingface.co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell)\n* [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n* [YOLOS](https://huggingface."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2ac61883-fb41-444a-b348-51ef0b96f637
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2ac61883-fb41-444a-b348-51ef0b96f637\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dc0829ba-b49e-4029-9cb2-e3feffc11f08\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d1c1382e87d8d16121b6e2e66a13d905bd5a7982e0ac79c4e44459c476b391e7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9d97cb5c-8f99-41ea-bebb-6b31b554cc28\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"426e9868cb1e0c0ea1825b024480ed7b6c2b96f96675b126f6049b8702d2f2f9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14314, \"end_char_idx\": 17113, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "co/docs/transformers/model_doc/vit#transformers.ViTModel)\n* [ViTHybrid](https://huggingface.co/docs/transformers/model_doc/vit_hybrid#transformers.ViTHybridModel)\n* [ViTMAE](https://huggingface.co/docs/transformers/model_doc/vit_mae#transformers.ViTMAEModel)\n* [ViTMSN](https://huggingface.co/docs/transformers/model_doc/vit_msn#transformers.ViTMSNModel)\n* [VideoMAE](https://huggingface.co/docs/transformers/model_doc/videomae#transformers.VideoMAEModell)\n* [wav2vec2](https://huggingface.co/docs/transformers/model_doc/wav2vec2#transformers.Wav2Vec2Model)\n* [Hubert](https://huggingface.co/docs/transformers/model_doc/hubert#transformers.HubertModel)\n* [data2vec_audio](https://huggingface.co/docs/transformers/main/en/model_doc/data2vec#transformers.Data2VecAudioModel)\n* [SigLIP](https://huggingface.co/docs/transformers/model_doc/siglip)\n* [Sew](https://huggingface.co/docs/transformers/main/en/model_doc/sew#transformers.SEWModel)\n* [UniSpeech](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech#transformers.UniSpeechModel)\n* [unispeech_sat](https://huggingface.co/docs/transformers/v4.39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n* [YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos#transformers.YolosModel)\n\n\n<Tip>\n\nFlashAttention can only be used for models with the `fp16` or `bf16` torch type, so make sure to cast your model to the appropriate type first. The memory-efficient attention backend is able to handle `fp32` models.\n\n</Tip>\n\n<Tip>\n\nSDPA does not support certain sets of attention parameters, such as `head_mask` and `output_attentions=True`.\nIn that case, you should see a warning message and we will fall back to the (slower) eager implementation.\n\n</Tip>\n\nBy default, SDPA selects the most performant kernel available but you can check whether a backend is available in a given setting (hardware, problem size) with [`torch.backends.cuda.sdp_kernel`](https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel) as a context manager:\n\n```diff\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.float16).to(\"cuda\")\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```\n\nIf you see a bug with the traceback below, try using the nightly version of PyTorch which may have broader coverage for FlashAttention:\n\n```bash\nRuntimeError: No available kernel. Aborting execution."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 9d97cb5c-8f99-41ea-bebb-6b31b554cc28
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9d97cb5c-8f99-41ea-bebb-6b31b554cc28\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2ac61883-fb41-444a-b348-51ef0b96f637\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c41e249f85de51b493e6bcd35f101c3e68238c37931acb17e79aab3bca29ca6b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"47873a84-430c-48b7-8f65-a7cce2992e81\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"28251a96f8e71bb53783ab61d813ea4db87890e499c3ab3716e7a4d765432973\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15428, \"end_char_idx\": 18818, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "39.3/en/model_doc/unispeech-sat#transformers.UniSpeechSatModel)\n* [YOLOS](https://huggingface.co/docs/transformers/model_doc/yolos#transformers.YolosModel)\n\n\n<Tip>\n\nFlashAttention can only be used for models with the `fp16` or `bf16` torch type, so make sure to cast your model to the appropriate type first. The memory-efficient attention backend is able to handle `fp32` models.\n\n</Tip>\n\n<Tip>\n\nSDPA does not support certain sets of attention parameters, such as `head_mask` and `output_attentions=True`.\nIn that case, you should see a warning message and we will fall back to the (slower) eager implementation.\n\n</Tip>\n\nBy default, SDPA selects the most performant kernel available but you can check whether a backend is available in a given setting (hardware, problem size) with [`torch.backends.cuda.sdp_kernel`](https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel) as a context manager:\n\n```diff\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", torch_dtype=torch.float16).to(\"cuda\")\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```\n\nIf you see a bug with the traceback below, try using the nightly version of PyTorch which may have broader coverage for FlashAttention:\n\n```bash\nRuntimeError: No available kernel. Aborting execution.\n\n# install PyTorch nightly\npip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118\n```\n\n## BetterTransformer\n\n<Tip warning={true}>\n\nSome BetterTransformer features are being upstreamed to Transformers with default support for native `torch.nn.scaled_dot_product_attention`. BetterTransformer still has a wider coverage than the Transformers SDPA integration, but you can expect more and more architectures to natively support SDPA in Transformers.\n\n</Tip>\n\n<Tip>\n\nCheck out our benchmarks with BetterTransformer and scaled dot product attention in the [Out of the box acceleration and memory savings of \ud83e\udd17 decoder models with PyTorch 2.0](https://pytorch.org/blog/out-of-the-box-acceleration/) and learn more about the fastpath execution in the [BetterTransformer](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2) blog post.\n\n</Tip>\n\nBetterTransformer accelerates inference with its fastpath (native PyTorch specialized implementation of Transformer functions) execution. The two optimizations in the fastpath execution are:\n\n1. fusion, which combines multiple sequential operations into a single \"kernel\" to reduce the number of computation steps\n2. skipping the inherent sparsity of padding tokens to avoid unnecessary computation with nested tensors\n\nBetterTransformer also converts all attention operations to use the more memory-efficient [scaled dot product attention (SDPA)](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention), and it calls optimized kernels like [FlashAttention](https://huggingface.co/papers/2205.14135) under the hood."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 47873a84-430c-48b7-8f65-a7cce2992e81
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"47873a84-430c-48b7-8f65-a7cce2992e81\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9d97cb5c-8f99-41ea-bebb-6b31b554cc28\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"26a2e59abdde7058008df070f8190639bc97d5524ca7b2a1bc51ff4a7fa735e3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3347ee4f-094f-4ba6-add0-25d07f830f24\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2df3ff132a0114795f51163b6ad5e6ad7b45a77c21d624270b9da3a6a3eaa32d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17094, \"end_char_idx\": 20461, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "Aborting execution.\n\n# install PyTorch nightly\npip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118\n```\n\n## BetterTransformer\n\n<Tip warning={true}>\n\nSome BetterTransformer features are being upstreamed to Transformers with default support for native `torch.nn.scaled_dot_product_attention`. BetterTransformer still has a wider coverage than the Transformers SDPA integration, but you can expect more and more architectures to natively support SDPA in Transformers.\n\n</Tip>\n\n<Tip>\n\nCheck out our benchmarks with BetterTransformer and scaled dot product attention in the [Out of the box acceleration and memory savings of \ud83e\udd17 decoder models with PyTorch 2.0](https://pytorch.org/blog/out-of-the-box-acceleration/) and learn more about the fastpath execution in the [BetterTransformer](https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2) blog post.\n\n</Tip>\n\nBetterTransformer accelerates inference with its fastpath (native PyTorch specialized implementation of Transformer functions) execution. The two optimizations in the fastpath execution are:\n\n1. fusion, which combines multiple sequential operations into a single \"kernel\" to reduce the number of computation steps\n2. skipping the inherent sparsity of padding tokens to avoid unnecessary computation with nested tensors\n\nBetterTransformer also converts all attention operations to use the more memory-efficient [scaled dot product attention (SDPA)](https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention), and it calls optimized kernels like [FlashAttention](https://huggingface.co/papers/2205.14135) under the hood.\n\nBefore you start, make sure you have \ud83e\udd17 Optimum [installed](https://huggingface.co/docs/optimum/installation).\n\nThen you can enable BetterTransformer with the [`PreTrainedModel.to_bettertransformer`] method:\n\n```python\nmodel = model.to_bettertransformer()\n```\n\nYou can return the original Transformers model with the [`~PreTrainedModel.reverse_bettertransformer`] method. You should use this before saving your model to use the canonical Transformers modeling:\n\n```py\nmodel = model.reverse_bettertransformer()\nmodel.save_pretrained(\"saved_model\")\n```\n\n## bitsandbytes\n\nbitsandbytes is a quantization library that includes support for 4-bit and 8-bit quantization. Quantization reduces your model size compared to its native full precision version, making it easier to fit large models onto GPUs with limited memory.\n\nMake sure you have bitsandbytes and \ud83e\udd17 Accelerate installed:\n\n```bash\n# these versions support 8-bit and 4-bit\npip install bitsandbytes>=0.39.0 accelerate>=0.20.0\n\n# install Transformers\npip install transformers\n```\n\n### 4-bit\n\nTo load a model in 4-bit for inference, use the `load_in_4bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"auto\", load_in_4bit=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 3347ee4f-094f-4ba6-add0-25d07f830f24
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3347ee4f-094f-4ba6-add0-25d07f830f24\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"47873a84-430c-48b7-8f65-a7cce2992e81\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ccf78299996d8a5758907ce69ddf30340b038e2f0ebbbfe4b4d09da24c7ce8b6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dabad0cf-e41d-4cc9-8d5a-79edfa600d7f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6422d4ce7f3d2b9b0e65a4424f086fa872764eb797f625898186291905e181eb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18820, \"end_char_idx\": 21976, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "Before you start, make sure you have \ud83e\udd17 Optimum [installed](https://huggingface.co/docs/optimum/installation).\n\nThen you can enable BetterTransformer with the [`PreTrainedModel.to_bettertransformer`] method:\n\n```python\nmodel = model.to_bettertransformer()\n```\n\nYou can return the original Transformers model with the [`~PreTrainedModel.reverse_bettertransformer`] method. You should use this before saving your model to use the canonical Transformers modeling:\n\n```py\nmodel = model.reverse_bettertransformer()\nmodel.save_pretrained(\"saved_model\")\n```\n\n## bitsandbytes\n\nbitsandbytes is a quantization library that includes support for 4-bit and 8-bit quantization. Quantization reduces your model size compared to its native full precision version, making it easier to fit large models onto GPUs with limited memory.\n\nMake sure you have bitsandbytes and \ud83e\udd17 Accelerate installed:\n\n```bash\n# these versions support 8-bit and 4-bit\npip install bitsandbytes>=0.39.0 accelerate>=0.20.0\n\n# install Transformers\npip install transformers\n```\n\n### 4-bit\n\nTo load a model in 4-bit for inference, use the `load_in_4bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment.\n\n```py\nfrom transformers import AutoModelForCausalLM\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"auto\", load_in_4bit=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 600MB of memory to the first GPU and 1GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"600MB\", 1: \"1GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_4bit=True, max_memory=max_memory_mapping\n)\n```\n\n### 8-bit\n\n<Tip>\n\nIf you're curious and interested in learning more about the concepts underlying 8-bit quantization, read the [Gentle Introduction to 8-bit Matrix Multiplication for transformers at scale using Hugging Face Transformers, Accelerate and bitsandbytes](https://huggingface.co/blog/hf-bitsandbytes-integration) blog post.\n\n</Tip>\n\nTo load a model in 8-bit for inference, use the `load_in_8bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment:\n\n```py\nfrom transformers import AutoModelForCausalLM, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n```\n\nIf you're loading a model in 8-bit for text generation, you should use the [`~transformers.GenerationMixin.generate`] method instead of the [`Pipeline`] function which is not optimized for 8-bit models and will be slower. Some sampling strategies, like nucleus sampling, are also not supported by the [`Pipeline`] for 8-bit models."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: dabad0cf-e41d-4cc9-8d5a-79edfa600d7f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"dabad0cf-e41d-4cc9-8d5a-79edfa600d7f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3347ee4f-094f-4ba6-add0-25d07f830f24\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c62a1c64e557e7a42fc7337f67169a4202ed6f934e57e516cdedaf9e81e1def7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fe16e2f0-17e3-4ed3-ae9b-dc669ec2963d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"60189b310d343e7254922d56587eb1b57d4cc9b5e2f60c25e867df0efffbed02\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20462, \"end_char_idx\": 23333, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "For example, to distribute 600MB of memory to the first GPU and 1GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"600MB\", 1: \"1GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_4bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_4bit=True, max_memory=max_memory_mapping\n)\n```\n\n### 8-bit\n\n<Tip>\n\nIf you're curious and interested in learning more about the concepts underlying 8-bit quantization, read the [Gentle Introduction to 8-bit Matrix Multiplication for transformers at scale using Hugging Face Transformers, Accelerate and bitsandbytes](https://huggingface.co/blog/hf-bitsandbytes-integration) blog post.\n\n</Tip>\n\nTo load a model in 8-bit for inference, use the `load_in_8bit` parameter. The `device_map` parameter is optional, but we recommend setting it to `\"auto\"` to allow \ud83e\udd17 Accelerate to automatically and efficiently allocate the model given the available resources in the environment:\n\n```py\nfrom transformers import AutoModelForCausalLM, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n```\n\nIf you're loading a model in 8-bit for text generation, you should use the [`~transformers.GenerationMixin.generate`] method instead of the [`Pipeline`] function which is not optimized for 8-bit models and will be slower. Some sampling strategies, like nucleus sampling, are also not supported by the [`Pipeline`] for 8-bit models. You should also place all inputs on the same device as the model:\n\n```py\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n\nprompt = \"Hello, my llama is cute\"\ninputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\ngenerated_ids = model.generate(**inputs)\noutputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 1GB of memory to the first GPU and 2GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"1GB\", 1: \"2GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_8bit=True, max_memory=max_memory_mapping\n)\n```\n\n<Tip>\n\nFeel free to try running a 11 billion parameter [T5 model](https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing) or the 3 billion parameter [BLOOM model](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing) for inference on Google Colab's free tier GPUs!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: fe16e2f0-17e3-4ed3-ae9b-dc669ec2963d
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"fe16e2f0-17e3-4ed3-ae9b-dc669ec2963d\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dabad0cf-e41d-4cc9-8d5a-79edfa600d7f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d8ddbf5e5db99983304993cda8d5e992757ccbc0f2df2fc3bba2c054b44cc723\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2982c02a-1451-4492-9bfe-53dff915c84c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4fb7fa09c09665776357c1f165b687645b85ad0ae4e9cc8bc300cb83ee1c2ff0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21977, \"end_char_idx\": 24848, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "You should also place all inputs on the same device as the model:\n\n```py\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\nmodel_name = \"bigscience/bloom-2b5\"\ntokenizer = AutoTokenizer.from_pretrained(model_name)\nmodel_8bit = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=BitsAndBytesConfig(load_in_8bit=True))\n\nprompt = \"Hello, my llama is cute\"\ninputs = tokenizer(prompt, return_tensors=\"pt\").to(\"cuda\")\ngenerated_ids = model.generate(**inputs)\noutputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)\n```\n\nTo load a model in 4-bit for inference with multiple GPUs, you can control how much GPU RAM you want to allocate to each GPU. For example, to distribute 1GB of memory to the first GPU and 2GB of memory to the second GPU:\n\n```py\nmax_memory_mapping = {0: \"1GB\", 1: \"2GB\"}\nmodel_name = \"bigscience/bloom-3b\"\nmodel_8bit = AutoModelForCausalLM.from_pretrained(\n    model_name, device_map=\"auto\", load_in_8bit=True, max_memory=max_memory_mapping\n)\n```\n\n<Tip>\n\nFeel free to try running a 11 billion parameter [T5 model](https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing) or the 3 billion parameter [BLOOM model](https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing) for inference on Google Colab's free tier GPUs!\n\n</Tip>\n\n## \ud83e\udd17 Optimum\n\n<Tip>\n\nLearn more details about using ORT with \ud83e\udd17 Optimum in the [Accelerated inference on NVIDIA GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides. This section only provides a brief and simple example.\n\n</Tip>\n\nONNX Runtime (ORT) is a model accelerator that supports accelerated inference on Nvidia GPUs, and AMD GPUs that use [ROCm](https://www.amd.com/en/products/software/rocm.html) stack. ORT uses optimization techniques like fusing common operations into a single node and constant folding to reduce the number of computations performed and speedup inference. ORT also places the most computationally intensive operations on the GPU and the rest on the CPU to intelligently distribute the workload between the two devices.\n\nORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider)."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2982c02a-1451-4492-9bfe-53dff915c84c
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2982c02a-1451-4492-9bfe-53dff915c84c\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fe16e2f0-17e3-4ed3-ae9b-dc669ec2963d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"08d1fb64820d984927e3a1e1675323d4b6f668d46cc69867911414352bb05fb3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f681a33f-b00c-4585-b69a-53aad2679959\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6983403900b43b4cde8e5634c3937a5cb5a14610c1a42b3f450fc8fa4bf4251a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23335, \"end_char_idx\": 25875, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "</Tip>\n\n## \ud83e\udd17 Optimum\n\n<Tip>\n\nLearn more details about using ORT with \ud83e\udd17 Optimum in the [Accelerated inference on NVIDIA GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#accelerated-inference-on-nvidia-gpus) and [Accelerated inference on AMD GPUs](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu#accelerated-inference-on-amd-gpus) guides. This section only provides a brief and simple example.\n\n</Tip>\n\nONNX Runtime (ORT) is a model accelerator that supports accelerated inference on Nvidia GPUs, and AMD GPUs that use [ROCm](https://www.amd.com/en/products/software/rocm.html) stack. ORT uses optimization techniques like fusing common operations into a single node and constant folding to reduce the number of computations performed and speedup inference. ORT also places the most computationally intensive operations on the GPU and the rest on the CPU to intelligently distribute the workload between the two devices.\n\nORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider). If you want to load a model that was not yet exported to ONNX, you can set `export=True` to convert your model on-the-fly to the ONNX format:\n\n```py\nfrom optimum.onnxruntime import ORTModelForSequenceClassification\n\nort_model = ORTModelForSequenceClassification.from_pretrained(\n  \"distilbert/distilbert-base-uncased-finetuned-sst-2-english\",\n  export=True,\n  provider=\"CUDAExecutionProvider\",\n)\n```\n\nNow you're free to use the model for inference:\n\n```py\nfrom optimum.pipelines import pipeline\nfrom transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased-finetuned-sst-2-english\")\n\npipeline = pipeline(task=\"text-classification\", model=ort_model, tokenizer=tokenizer, device=\"cuda:0\")\nresult = pipeline(\"Both the music and visual were astounding, not to mention the actors performance.\")\n```\n\n## Combine optimizations\n\nIt is often possible to combine several of the optimization techniques described above to get the best inference performance possible for your model."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f681a33f-b00c-4585-b69a-53aad2679959
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_infer_gpu_one", "title": "GPU inference", "tokens": 7025, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f681a33f-b00c-4585-b69a-53aad2679959\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"4f7216d6-043c-4dda-a6ef-226e01ba6879\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"007d040f9c76352423c4d8e3f06a3eb050c8dc8a866aaf2368a104a0b1f96873\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2982c02a-1451-4492-9bfe-53dff915c84c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_infer_gpu_one\", \"title\": \"GPU inference\", \"tokens\": 7025, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"50ef78b03e735568e2f68439af972426ab19539fc50f0484a99945591d1e9f20\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24297, \"end_char_idx\": 26780, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "ref_doc_id": "4f7216d6-043c-4dda-a6ef-226e01ba6879", "chroma:document": "ORT is supported by \ud83e\udd17 Optimum which can be used in \ud83e\udd17 Transformers. You'll need to use an [`~optimum.onnxruntime.ORTModel`] for the task you're solving, and specify the `provider` parameter which can be set to either [`CUDAExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#cudaexecutionprovider), [`ROCMExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/amdgpu) or [`TensorrtExecutionProvider`](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/gpu#tensorrtexecutionprovider). If you want to load a model that was not yet exported to ONNX, you can set `export=True` to convert your model on-the-fly to the ONNX format:\n\n```py\nfrom optimum.onnxruntime import ORTModelForSequenceClassification\n\nort_model = ORTModelForSequenceClassification.from_pretrained(\n  \"distilbert/distilbert-base-uncased-finetuned-sst-2-english\",\n  export=True,\n  provider=\"CUDAExecutionProvider\",\n)\n```\n\nNow you're free to use the model for inference:\n\n```py\nfrom optimum.pipelines import pipeline\nfrom transformers import AutoTokenizer\n\ntokenizer = AutoTokenizer.from_pretrained(\"distilbert/distilbert-base-uncased-finetuned-sst-2-english\")\n\npipeline = pipeline(task=\"text-classification\", model=ort_model, tokenizer=tokenizer, device=\"cuda:0\")\nresult = pipeline(\"Both the music and visual were astounding, not to mention the actors performance.\")\n```\n\n## Combine optimizations\n\nIt is often possible to combine several of the optimization techniques described above to get the best inference performance possible for your model. For example, you can load a model in 4-bit, and then enable BetterTransformer with FlashAttention:\n\n```py\nimport torch\nfrom transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig\n\n# load model in 4-bit\nquantization_config = BitsAndBytesConfig(\n    load_in_4bit=True,\n    bnb_4bit_compute_dtype=torch.float16\n)\n\ntokenizer = AutoTokenizer.from_pretrained(\"facebook/opt-350m\")\nmodel = AutoModelForCausalLM.from_pretrained(\"facebook/opt-350m\", quantization_config=quantization_config)\n\n# enable BetterTransformer\nmodel = model.to_bettertransformer()\n\ninput_text = \"Hello my dog is cute and\"\ninputs = tokenizer(input_text, return_tensors=\"pt\").to(\"cuda\")\n\n# enable FlashAttention\nwith torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):\n    outputs = model.generate(**inputs)\n\nprint(tokenizer.decode(outputs[0], skip_special_tokens=True))\n```"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 9ee218da-d410-4a34-ba2b-5c7a1c40abdb
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9ee218da-d410-4a34-ba2b-5c7a1c40abdb\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f2f58414-f07a-48c8-9ef8-0d011baf1941\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9ccfbd280615fea0c7e9ac62bd7ee5551db95b730ecddcfac28f222f8a580983\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2738, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "# Efficient Training on Multiple CPUs\n\nWhen training on a single CPU is too slow, we can use multiple CPUs. This guide focuses on PyTorch-based DDP enabling\ndistributed CPU training efficiently on [bare metal](#usage-in-trainer) and [Kubernetes](#usage-with-kubernetes).\n\n## Intel\u00ae oneCCL Bindings for PyTorch\n\n[Intel\u00ae oneCCL](https://github.com/oneapi-src/oneCCL) (collective communications library) is a library for efficient distributed deep learning training implementing such collectives like allreduce, allgather, alltoall. For more information on oneCCL, please refer to the [oneCCL documentation](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html) and [oneCCL specification](https://spec.oneapi.com/versions/latest/elements/oneCCL/source/index.html).\n\nModule `oneccl_bindings_for_pytorch` (`torch_ccl` before version 1.12)  implements PyTorch C10D ProcessGroup API and can be dynamically loaded as external ProcessGroup and only works on Linux platform now\n\nCheck more detailed information for [oneccl_bind_pt](https://github.com/intel/torch-ccl).\n\n### Intel\u00ae oneCCL Bindings for PyTorch installation\n\nWheel files are available for the following Python versions:\n\n| Extension Version | Python 3.6 | Python 3.7 | Python 3.8 | Python 3.9 | Python 3.10 |\n| :---------------: | :--------: | :--------: | :--------: | :--------: | :---------: |\n| 2.1.0             |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 2.0.0             |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.13.0            |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.12.100          |            | \u221a          | \u221a          | \u221a          | \u221a           |\n| 1.12.0            |            | \u221a          | \u221a          | \u221a          | \u221a           |\n\nPlease run `pip list | grep torch` to get your `pytorch_version`.\n```bash\npip install oneccl_bind_pt=={pytorch_version} -f https://developer.intel.com/ipex-whl-stable-cpu\n```\nwhere `{pytorch_version}` should be your PyTorch version, for instance 2.1.0.\nCheck more approaches for [oneccl_bind_pt installation](https://github.com/intel/torch-ccl).\nVersions of oneCCL and PyTorch must match.\n\n<Tip warning={true}>\n\noneccl_bindings_for_pytorch 1.12.0 prebuilt wheel does not work with PyTorch 1.12.1 (it is for PyTorch 1.12.0)\nPyTorch 1.12.1 should work with oneccl_bindings_for_pytorch 1.12.100\n\n</Tip>\n\n## Intel\u00ae MPI library\nUse this standards-based MPI implementation to deliver flexible, efficient, scalable cluster messaging on Intel\u00ae architecture. This component is part of the Intel\u00ae oneAPI HPC Toolkit.\n\noneccl_bindings_for_pytorch is installed along with the MPI tool set. Need to source the environment before using it."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f2f58414-f07a-48c8-9ef8-0d011baf1941
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f2f58414-f07a-48c8-9ef8-0d011baf1941\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9ee218da-d410-4a34-ba2b-5c7a1c40abdb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"6bb1eb5d523116886edf61afc0047f62889ba241c5bb59e9665c074368926d3c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3c3d8c16-1835-4ce9-ab47-ef0475f2dca7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cc42a1766cb4ac5780371b8b51f5518e1f7b69ad5217ba383aec1bfae4ddac46\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1879, \"end_char_idx\": 4762, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "```bash\npip install oneccl_bind_pt=={pytorch_version} -f https://developer.intel.com/ipex-whl-stable-cpu\n```\nwhere `{pytorch_version}` should be your PyTorch version, for instance 2.1.0.\nCheck more approaches for [oneccl_bind_pt installation](https://github.com/intel/torch-ccl).\nVersions of oneCCL and PyTorch must match.\n\n<Tip warning={true}>\n\noneccl_bindings_for_pytorch 1.12.0 prebuilt wheel does not work with PyTorch 1.12.1 (it is for PyTorch 1.12.0)\nPyTorch 1.12.1 should work with oneccl_bindings_for_pytorch 1.12.100\n\n</Tip>\n\n## Intel\u00ae MPI library\nUse this standards-based MPI implementation to deliver flexible, efficient, scalable cluster messaging on Intel\u00ae architecture. This component is part of the Intel\u00ae oneAPI HPC Toolkit.\n\noneccl_bindings_for_pytorch is installed along with the MPI tool set. Need to source the environment before using it.\n\nfor Intel\u00ae oneCCL >= 1.12.0\n```bash\noneccl_bindings_for_pytorch_path=$(python -c \"from oneccl_bindings_for_pytorch import cwd; print(cwd)\")\nsource $oneccl_bindings_for_pytorch_path/env/setvars.sh\n```\n\nfor Intel\u00ae oneCCL whose version < 1.12.0\n```bash\ntorch_ccl_path=$(python -c \"import torch; import torch_ccl; import os;  print(os.path.abspath(os.path.dirname(torch_ccl.__file__)))\")\nsource $torch_ccl_path/env/setvars.sh\n```\n\n#### Intel\u00ae Extension for PyTorch installation\n\nIntel Extension for PyTorch (IPEX) provides performance optimizations for CPU training with both Float32 and BFloat16 (refer to the [single CPU section](./perf_train_cpu) to learn more).\n\n\nThe following \"Usage in Trainer\" takes mpirun in Intel\u00ae MPI library as an example.\n\n\n## Usage in Trainer\nTo enable multi CPU distributed training in the Trainer with the ccl backend, users should add **`--ddp_backend ccl`** in the command arguments.\n\nLet's see an example with the [question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)\n\n\nThe following command enables training with 2 processes on one Xeon node, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=127.0.0.1\n mpirun -n 2 -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex\n```\nThe following command enables training with a total of four processes on two Xeons (node0 and node1, taking node0 as the main process), ppn (processes per node) is set to 2, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 3c3d8c16-1835-4ce9-ab47-ef0475f2dca7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3c3d8c16-1835-4ce9-ab47-ef0475f2dca7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f2f58414-f07a-48c8-9ef8-0d011baf1941\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f946e61e467eaa0746476bd4f4152c19cc2a43a27bda090ba1a9b57b3c4a3c94\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0d12b0bd-b297-48f3-b831-6f320ce22eda\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"99fe64b58df5888dd57dfbd3772bd28606c3155ef518407a7570183ade8f452d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3404, \"end_char_idx\": 6320, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "The following \"Usage in Trainer\" takes mpirun in Intel\u00ae MPI library as an example.\n\n\n## Usage in Trainer\nTo enable multi CPU distributed training in the Trainer with the ccl backend, users should add **`--ddp_backend ccl`** in the command arguments.\n\nLet's see an example with the [question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)\n\n\nThe following command enables training with 2 processes on one Xeon node, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=127.0.0.1\n mpirun -n 2 -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex\n```\nThe following command enables training with a total of four processes on two Xeons (node0 and node1, taking node0 as the main process), ppn (processes per node) is set to 2, with one process running per one socket. The variables OMP_NUM_THREADS/CCL_WORKER_COUNT can be tuned for optimal performance.\n\nIn node0, you need to create a configuration file which contains the IP addresses of each node (for example hostfile) and pass that configuration file path as an argument.\n```shell script\n cat hostfile\n xxx.xxx.xxx.xxx #node0 ip\n xxx.xxx.xxx.xxx #node1 ip\n```\nNow, run the following command in node0 and **4DDP** will be enabled in node0 and node1 with BF16 auto mixed precision:\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=xxx.xxx.xxx.xxx #node0 ip\n mpirun -f hostfile -n 4 -ppn 2 \\\n -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex \\\n --bf16\n```\n\n## Usage with Kubernetes\n\nThe same distributed training job from the previous section can be deployed to a Kubernetes cluster using the\n[Kubeflow PyTorchJob training operator](https://www.kubeflow.org/docs/components/training/pytorch/).\n\n### Setup\n\nThis example assumes that you have:\n* Access to a Kubernetes cluster with [Kubeflow installed](https://www.kubeflow.org/docs/started/installing-kubeflow/)\n* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) installed and configured to access the Kubernetes cluster\n* A [Persistent Volume Claim (PVC)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) that can be used\n  to store datasets and model files."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 0d12b0bd-b297-48f3-b831-6f320ce22eda
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0d12b0bd-b297-48f3-b831-6f320ce22eda\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c3d8c16-1835-4ce9-ab47-ef0475f2dca7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b49cc5d5f3b618d9cd15e04f8c99c98c7d2869aab6bf823947d375bf7ab1a12a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"130cffc2-d372-46e1-bdf7-49162e6593b4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7a07ab00c96e38bf3763e47a08be9a7bd2da43ff46e947acaae17e8f5d1bf8b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4936, \"end_char_idx\": 7756, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "```shell script\n cat hostfile\n xxx.xxx.xxx.xxx #node0 ip\n xxx.xxx.xxx.xxx #node1 ip\n```\nNow, run the following command in node0 and **4DDP** will be enabled in node0 and node1 with BF16 auto mixed precision:\n```shell script\n export CCL_WORKER_COUNT=1\n export MASTER_ADDR=xxx.xxx.xxx.xxx #node0 ip\n mpirun -f hostfile -n 4 -ppn 2 \\\n -genv OMP_NUM_THREADS=23 \\\n python3 run_qa.py \\\n --model_name_or_path google-bert/bert-large-uncased \\\n --dataset_name squad \\\n --do_train \\\n --do_eval \\\n --per_device_train_batch_size 12  \\\n --learning_rate 3e-5  \\\n --num_train_epochs 2  \\\n --max_seq_length 384 \\\n --doc_stride 128  \\\n --output_dir /tmp/debug_squad/ \\\n --no_cuda \\\n --ddp_backend ccl \\\n --use_ipex \\\n --bf16\n```\n\n## Usage with Kubernetes\n\nThe same distributed training job from the previous section can be deployed to a Kubernetes cluster using the\n[Kubeflow PyTorchJob training operator](https://www.kubeflow.org/docs/components/training/pytorch/).\n\n### Setup\n\nThis example assumes that you have:\n* Access to a Kubernetes cluster with [Kubeflow installed](https://www.kubeflow.org/docs/started/installing-kubeflow/)\n* [`kubectl`](https://kubernetes.io/docs/tasks/tools/) installed and configured to access the Kubernetes cluster\n* A [Persistent Volume Claim (PVC)](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) that can be used\n  to store datasets and model files. There are multiple options for setting up the PVC including using an NFS\n  [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/) or a cloud storage bucket.\n* A Docker container that includes your model training script and all the dependencies needed to run the script. For\n  distributed CPU training jobs, this typically includes PyTorch, Transformers, Intel Extension for PyTorch, Intel\n  oneCCL Bindings for PyTorch, and OpenSSH to communicate between the containers.\n\nThe snippet below is an example of a Dockerfile that uses a base image that supports distributed CPU training and then\nextracts a Transformers release to the `/workspace` directory, so that the example scripts are included in the image:\n```dockerfile\nFROM intel/ai-workflows:torch-2.0.1-huggingface-multinode-py3.9\n\nWORKDIR /workspace\n\n# Download and extract the transformers code\nARG HF_TRANSFORMERS_VER=\"4.35.2\"\nRUN mkdir transformers && \\\n    curl -sSL --retry 5 https://github.com/huggingface/transformers/archive/refs/tags/v${HF_TRANSFORMERS_VER}.tar.gz | tar -C transformers --strip-components=1 -xzf -\n```\nThe image needs to be built and copied to the cluster's nodes or pushed to a container registry prior to deploying the\nPyTorchJob to the cluster.\n\n### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 130cffc2-d372-46e1-bdf7-49162e6593b4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"130cffc2-d372-46e1-bdf7-49162e6593b4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0d12b0bd-b297-48f3-b831-6f320ce22eda\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"16b0b2f89859c4c187e5284de3a273abbc0e9a1084b0cfff38360346c46c4ec2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ad484ac9-60a3-459f-bb93-e9c064f793e1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f5d3deedbae3471e7fb8baf160bbf0e933862fb86a7a668fb1d0b1a9f9babbe4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6321, \"end_char_idx\": 8573, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "There are multiple options for setting up the PVC including using an NFS\n  [storage class](https://kubernetes.io/docs/concepts/storage/storage-classes/) or a cloud storage bucket.\n* A Docker container that includes your model training script and all the dependencies needed to run the script. For\n  distributed CPU training jobs, this typically includes PyTorch, Transformers, Intel Extension for PyTorch, Intel\n  oneCCL Bindings for PyTorch, and OpenSSH to communicate between the containers.\n\nThe snippet below is an example of a Dockerfile that uses a base image that supports distributed CPU training and then\nextracts a Transformers release to the `/workspace` directory, so that the example scripts are included in the image:\n```dockerfile\nFROM intel/ai-workflows:torch-2.0.1-huggingface-multinode-py3.9\n\nWORKDIR /workspace\n\n# Download and extract the transformers code\nARG HF_TRANSFORMERS_VER=\"4.35.2\"\nRUN mkdir transformers && \\\n    curl -sSL --retry 5 https://github.com/huggingface/transformers/archive/refs/tags/v${HF_TRANSFORMERS_VER}.tar.gz | tar -C transformers --strip-components=1 -xzf -\n```\nThe image needs to be built and copied to the cluster's nodes or pushed to a container registry prior to deploying the\nPyTorchJob to the cluster.\n\n### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster. The yaml file for the PyTorchJob defines parameters such as:\n * The name of the PyTorchJob\n * The number of replicas (workers)\n * The python script and it's parameters that will be used to run the training job\n * The types of resources (node selector, memory, and CPU) needed for each worker\n * The image/tag for the Docker container to use\n * Environment variables\n * A volume mount for the PVC\n\nThe volume mount defines a path where the PVC will be mounted in the container for each worker pod. This location can be\nused for the dataset, checkpoint files, and the saved model after training completes.\n\nThe snippet below is an example of a yaml file for a PyTorchJob with 4 workers running the\n[question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering)."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: ad484ac9-60a3-459f-bb93-e9c064f793e1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"ad484ac9-60a3-459f-bb93-e9c064f793e1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"130cffc2-d372-46e1-bdf7-49162e6593b4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"8c872e9008c5b52155bc25ad47f409e0b937d5963cb2792801ce8c4e9877c8d1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ae963237-20ce-4387-bedb-4f8541a372fa\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"94c7aa54c47772b3eb2781bd8e9ea38605eda9098a14a43dcfdd75d39f6b01be\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7576, \"end_char_idx\": 11580, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "### PyTorchJob Specification File\n\nThe [Kubeflow PyTorchJob](https://www.kubeflow.org/docs/components/training/pytorch/) is used to run the distributed\ntraining job on the cluster. The yaml file for the PyTorchJob defines parameters such as:\n * The name of the PyTorchJob\n * The number of replicas (workers)\n * The python script and it's parameters that will be used to run the training job\n * The types of resources (node selector, memory, and CPU) needed for each worker\n * The image/tag for the Docker container to use\n * Environment variables\n * A volume mount for the PVC\n\nThe volume mount defines a path where the PVC will be mounted in the container for each worker pod. This location can be\nused for the dataset, checkpoint files, and the saved model after training completes.\n\nThe snippet below is an example of a yaml file for a PyTorchJob with 4 workers running the\n[question-answering example](https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering).\n```yaml\napiVersion: \"kubeflow.org/v1\"\nkind: PyTorchJob\nmetadata:\n  name: transformers-pytorchjob\n  namespace: kubeflow\nspec:\n  elasticPolicy:\n    rdzvBackend: c10d\n    minReplicas: 1\n    maxReplicas: 4\n    maxRestarts: 10\n  pytorchReplicaSpecs:\n    Worker:\n      replicas: 4  # The number of worker pods\n      restartPolicy: OnFailure\n      template:\n        spec:\n          containers:\n            - name: pytorch\n              image: <image name>:<tag>  # Specify the docker image to use for the worker pods\n              imagePullPolicy: IfNotPresent\n              command:\n                - torchrun\n                - /workspace/transformers/examples/pytorch/question-answering/run_qa.py\n                - --model_name_or_path\n                - \"google-bert/bert-large-uncased\"\n                - --dataset_name\n                - \"squad\"\n                - --do_train\n                - --do_eval\n                - --per_device_train_batch_size\n                - \"12\"\n                - --learning_rate\n                - \"3e-5\"\n                - --num_train_epochs\n                - \"2\"\n                - --max_seq_length\n                - \"384\"\n                - --doc_stride\n                - \"128\"\n                - --output_dir\n                - \"/tmp/pvc-mount/output\"\n                - --no_cuda\n                - --ddp_backend\n                - \"ccl\"\n                - --use_ipex\n                - --bf16  # Specify --bf16 if your hardware supports bfloat16\n              env:\n              - name: LD_PRELOAD\n                value: \"/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4.5.9:/usr/local/lib/libiomp5.so\"\n              - name: TRANSFORMERS_CACHE\n                value: \"/tmp/pvc-mount/transformers_cache\"\n              - name: HF_DATASETS_CACHE\n                value: \"/tmp/pvc-mount/hf_datasets_cache\"\n              - name: LOGLEVEL\n                value: \"INFO\"\n              - name: CCL_WORKER_COUNT\n                value: \"1\"\n              - name: OMP_NUM_THREADS  # Can be tuned for optimal performance\n-                value: \"56\"\n              resources:\n                limits:\n                  cpu: 200  # Update the CPU and memory limit values based on your nodes\n                  memory: 128Gi\n                requests:\n                  cpu: 200  # Update the CPU and memory request values based on your nodes\n                  memory: 128Gi\n              volumeMounts:\n              - name: pvc-volume\n                mountPath: /tmp/pvc-mount\n              - mountPath: /dev/shm\n                name: dshm\n          restartPolicy: Never\n          nodeSelector:  #  Optionally use the node selector to specify what types of nodes to use for the workers\n            node-type: spr\n          volumes:\n          - name: pvc-volume\n            persistentVolumeClaim:\n              claimName: transformers-pvc\n          - name: dshm\n            emptyDir:\n              medium: Memory\n```\nTo run this example, update the yaml based on your training script and the nodes in your cluster."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: ae963237-20ce-4387-bedb-4f8541a372fa
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/perf_train_cpu_many", "title": "Efficient Training on Multiple CPUs", "tokens": 3677, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"ae963237-20ce-4387-bedb-4f8541a372fa\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"35ed512f-ca54-45b8-a5a6-eb5912c84e71\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"ffe609585380248d5b5108273a0d4d0feb0aa3c4bef6475e1078e180a1d4e440\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ad484ac9-60a3-459f-bb93-e9c064f793e1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/perf_train_cpu_many\", \"title\": \"Efficient Training on Multiple CPUs\", \"tokens\": 3677, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"05cea7679a80f168ce4da1837c8bcb923245531a706a6c0c678cd1c0b1e86c1b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11582, \"end_char_idx\": 14333, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "ref_doc_id": "35ed512f-ca54-45b8-a5a6-eb5912c84e71", "chroma:document": "<Tip>\n\nThe CPU resource limits/requests in the yaml are defined in [cpu units](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-cpu)\nwhere 1 CPU unit is equivalent to 1 physical CPU core or 1 virtual core (depending on whether the node is a physical\nhost or a VM). The amount of CPU and memory limits/requests defined in the yaml should be less than the amount of\navailable CPU/memory capacity on a single machine. It is usually a good idea to not use the entire machine's capacity in\norder to leave some resources for the kubelet and OS. In order to get [\"guaranteed\"](https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/#guaranteed)\n[quality of service](https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/) for the worker pods,\nset the same CPU and memory amounts for both the resource limits and requests.\n\n</Tip>\n\n### Deploy\n\nAfter the PyTorchJob spec has been updated with values appropriate for your cluster and training job, it can be deployed\nto the cluster using:\n```bash\nkubectl create -f pytorchjob.yaml\n```\n\nThe `kubectl get pods -n kubeflow` command can then be used to list the pods in the `kubeflow` namespace. You should see\nthe worker pods for the PyTorchJob that was just deployed. At first, they will probably have a status of \"Pending\" as\nthe containers get pulled and created, then the status should change to \"Running\".\n```\nNAME                                                     READY   STATUS                  RESTARTS          AGE\n...\ntransformers-pytorchjob-worker-0                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-1                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-2                         1/1     Running                 0                 7m37s\ntransformers-pytorchjob-worker-3                         1/1     Running                 0                 7m37s\n...\n```\n\nThe logs for worker can be viewed using `kubectl logs -n kubeflow <pod name>`. Add `-f` to stream the logs, for example:\n```bash\nkubectl logs -n kubeflow transformers-pytorchjob-worker-0 -f\n```\n\nAfter the training job completes, the trained model can be copied from the PVC or storage location. When you are done\nwith the job, the PyTorchJob resource can be deleted from the cluster using `kubectl delete -f pytorchjob.yaml`.\n\n## Summary\n\nThis guide covered running distributed PyTorch training jobs using multiple CPUs on bare metal and on a Kubernetes\ncluster. Both cases utilize Intel Extension for PyTorch and Intel oneCCL Bindings for PyTorch for optimal training\nperformance, and can be used as a template to run your own workload on multiple nodes."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 5bc13f53-148e-4043-892d-6e3661e8d12b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/pad_truncation", "title": "Padding and truncation", "tokens": 1364, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5bc13f53-148e-4043-892d-6e3661e8d12b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"123ca229-c652-4c30-8324-1ee8a14ff6ad\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"fcbf39d53d83c6f064cccf34a7654672d76b41edb3c0d340b128169d06afb319\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a4a6847a-2773-4103-8538-3d34af47b9ba\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e470205c927e63b5257be0726a9e40a2cb53fc05f060c8e73d6a930dc7d75c45\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3168, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "doc_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "ref_doc_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "chroma:document": "# Padding and truncation\n\nBatched inputs are often different lengths, so they can't be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special **padding token** to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences.\n\nIn most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: `padding`, `truncation` and `max_length`.\n\nThe `padding` argument controls padding. It can be a boolean or a string:\n\n  - `True` or `'longest'`: pad to the longest sequence in the batch (no padding is applied if you only provide\n    a single sequence).\n  - `'max_length'`: pad to a length specified by the `max_length` argument or the maximum length accepted\n    by the model if no `max_length` is provided (`max_length=None`). Padding will still be applied if you only provide a single sequence.\n  - `False` or `'do_not_pad'`: no padding is applied. This is the default behavior.\n\nThe `truncation` argument controls truncation. It can be a boolean or a string:\n\n  - `True` or `'longest_first'`: truncate to a maximum length specified by the `max_length` argument or\n    the maximum length accepted by the model if no `max_length` is provided (`max_length=None`). This will\n    truncate token by token, removing a token from the longest sequence in the pair until the proper length is\n    reached.\n  - `'only_second'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `'only_first'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `False` or `'do_not_truncate'`: no truncation is applied. This is the default behavior.\n\nThe `max_length` argument controls the length of the padding and truncation. It can be an integer or `None`, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to `max_length` is deactivated.\n\nThe following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace `truncation=True` by a `STRATEGY` selected in\n`['only_first', 'only_second', 'longest_first']`, i.e. `truncation='only_second'` or `truncation='longest_first'` to control how both sequences in the pair are truncated as detailed before."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: a4a6847a-2773-4103-8538-3d34af47b9ba
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/pad_truncation", "title": "Padding and truncation", "tokens": 1364, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"a4a6847a-2773-4103-8538-3d34af47b9ba\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"123ca229-c652-4c30-8324-1ee8a14ff6ad\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"fcbf39d53d83c6f064cccf34a7654672d76b41edb3c0d340b128169d06afb319\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5bc13f53-148e-4043-892d-6e3661e8d12b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/pad_truncation\", \"title\": \"Padding and truncation\", \"tokens\": 1364, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c63f8575ca6beac956d1366d237eb2a23446ea0293863d38ffdcf7640921b0a2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1613, \"end_char_idx\": 6842, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "doc_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "ref_doc_id": "123ca229-c652-4c30-8324-1ee8a14ff6ad", "chroma:document": "This will\n    truncate token by token, removing a token from the longest sequence in the pair until the proper length is\n    reached.\n  - `'only_second'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `'only_first'`: truncate to a maximum length specified by the `max_length` argument or the maximum\n    length accepted by the model if no `max_length` is provided (`max_length=None`). This will only truncate\n    the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.\n  - `False` or `'do_not_truncate'`: no truncation is applied. This is the default behavior.\n\nThe `max_length` argument controls the length of the padding and truncation. It can be an integer or `None`, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to `max_length` is deactivated.\n\nThe following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace `truncation=True` by a `STRATEGY` selected in\n`['only_first', 'only_second', 'longest_first']`, i.e. `truncation='only_second'` or `truncation='longest_first'` to control how both sequences in the pair are truncated as detailed before.\n\n| Truncation                           | Padding                           | Instruction                                                                                 |\n|--------------------------------------|-----------------------------------|---------------------------------------------------------------------------------------------|\n| no truncation                        | no padding                        | `tokenizer(batch_sentences)`                                                           |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True)` or                                          |\n|                                      |                                   | `tokenizer(batch_sentences, padding='longest')`                                        |\n|                                      | padding to max model input length | `tokenizer(batch_sentences, padding='max_length')`                                     |\n|                                      | padding to specific length        | `tokenizer(batch_sentences, padding='max_length', max_length=42)`                      |\n|                                      | padding to a multiple of a value  | `tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)`                        |\n| truncation to max model input length | no padding                        | `tokenizer(batch_sentences, truncation=True)` or                                       |\n|                                      |                                   | `tokenizer(batch_sentences, truncation=STRATEGY)`                                      |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True, truncation=True)` or                         |\n|                                      |                                   | `tokenizer(batch_sentences, padding=True, truncation=STRATEGY)`                        |\n|                                      | padding to max model input length | `tokenizer(batch_sentences, padding='max_length', truncation=True)` or                 |\n|                                      |                                   | `tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY)`                |\n|                                      | padding to specific length        | Not possible                                                                                |\n| truncation to specific length        | no padding                        | `tokenizer(batch_sentences, truncation=True, max_length=42)` or                        |\n|                                      |                                   | `tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)`                       |\n|                                      | padding to max sequence in batch  | `tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)` or          |\n|                                      |                                   | `tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)`         |\n|                                      | padding to max model input length | Not possible                                                                                |\n|                                      | padding to specific length        | `tokenizer(batch_sentences, padding='max_length', truncation=True, max_length=42)` or  |\n|                                      |                                   | `tokenizer(batch_sentences, padding='max_length', truncation=STRATEGY, max_length=42)` |"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 66c67af1-d146-4915-8d00-fafd48542201
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/accelerate", "title": "Distributed training with \ud83e\udd17 Accelerate", "tokens": 1054, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"66c67af1-d146-4915-8d00-fafd48542201\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"bd77a0d1-124f-4878-a2dc-d6fc9723d0f8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"839d0123fd4e6631f69648ba6ca5163ea5abc3628df918cbee14aaa6ef50dfd4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4e395ee4-7a36-4e96-876a-5bd5362d047a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d72d682e22186a9be416d73902d80574c8e1bb97898d735edede859bfb0be1ae\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3394, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "doc_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "ref_doc_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "chroma:document": "# Distributed training with \ud83e\udd17 Accelerate\n\nAs models get bigger, parallelism has emerged as a strategy for training larger models on limited hardware and accelerating training speed by several orders of magnitude. At Hugging Face, we created the [\ud83e\udd17 Accelerate](https://huggingface.co/docs/accelerate) library to help users easily train a \ud83e\udd17 Transformers model on any type of distributed setup, whether it is multiple GPU's on one machine or multiple GPU's across several machines. In this tutorial, learn how to customize your native PyTorch training loop to enable training in a distributed environment.\n\n## Setup\n\nGet started by installing \ud83e\udd17 Accelerate:\n\n```bash\npip install accelerate\n```\n\nThen import and create an [`~accelerate.Accelerator`] object. The [`~accelerate.Accelerator`] will automatically detect your type of distributed setup and initialize all the necessary components for training. You don't need to explicitly place your model on a device.\n\n```py\n>>> from accelerate import Accelerator\n\n>>> accelerator = Accelerator()\n```\n\n## Prepare to accelerate\n\nThe next step is to pass all the relevant training objects to the [`~accelerate.Accelerator.prepare`] method. This includes your training and evaluation DataLoaders, a model and an optimizer:\n\n```py\n>>> train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n...     train_dataloader, eval_dataloader, model, optimizer\n... )\n```\n\n## Backward\n\nThe last addition is to replace the typical `loss.backward()` in your training loop with \ud83e\udd17 Accelerate's [`~accelerate.Accelerator.backward`]method:\n\n```py\n>>> for epoch in range(num_epochs):\n...     for batch in train_dataloader:\n...         outputs = model(**batch)\n...         loss = outputs.loss\n...         accelerator.backward(loss)\n\n...         optimizer.step()\n...         lr_scheduler.step()\n...         optimizer.zero_grad()\n...         progress_bar.update(1)\n```\n\nAs you can see in the following code, you only need to add four additional lines of code to your training loop to enable distributed training!\n\n```diff\n+ from accelerate import Accelerator\n  from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler\n\n+ accelerator = Accelerator()\n\n  model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n  optimizer = AdamW(model.parameters(), lr=3e-5)\n\n- device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n- model.to(device)\n\n+ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n+     train_dataloader, eval_dataloader, model, optimizer\n+ )\n\n  num_epochs = 3\n  num_training_steps = num_epochs * len(train_dataloader)\n  lr_scheduler = get_scheduler(\n      \"linear\",\n      optimizer=optimizer,\n      num_warmup_steps=0,\n      num_training_steps=num_training_steps\n  )\n\n  progress_bar = tqdm(range(num_training_steps))\n\n  model.train()\n  for epoch in range(num_epochs):\n      for batch in train_dataloader:\n-         batch = {k: v.to(device) for k, v in batch.items()}\n          outputs = model(**batch)\n          loss = outputs.loss\n-         loss.backward()\n+         accelerator.backward(loss)\n\n          optimizer.step()\n          lr_scheduler.step()\n          optimizer.zero_grad()\n          progress_bar.update(1)\n```\n\n## Train\n\nOnce you've added the relevant lines of code, launch your training in a script or a notebook like Colaboratory."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 4e395ee4-7a36-4e96-876a-5bd5362d047a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/accelerate", "title": "Distributed training with \ud83e\udd17 Accelerate", "tokens": 1054, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4e395ee4-7a36-4e96-876a-5bd5362d047a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"bd77a0d1-124f-4878-a2dc-d6fc9723d0f8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"839d0123fd4e6631f69648ba6ca5163ea5abc3628df918cbee14aaa6ef50dfd4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"66c67af1-d146-4915-8d00-fafd48542201\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/accelerate\", \"title\": \"Distributed training with \\ud83e\\udd17 Accelerate\", \"tokens\": 1054, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"c14a49a78c6c1489bc7fb3bf51ebb898dcffbad5148dd111f4f49db77c336be0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2049, \"end_char_idx\": 4096, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "doc_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "ref_doc_id": "bd77a0d1-124f-4878-a2dc-d6fc9723d0f8", "chroma:document": "```diff\n+ from accelerate import Accelerator\n  from transformers import AdamW, AutoModelForSequenceClassification, get_scheduler\n\n+ accelerator = Accelerator()\n\n  model = AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=2)\n  optimizer = AdamW(model.parameters(), lr=3e-5)\n\n- device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n- model.to(device)\n\n+ train_dataloader, eval_dataloader, model, optimizer = accelerator.prepare(\n+     train_dataloader, eval_dataloader, model, optimizer\n+ )\n\n  num_epochs = 3\n  num_training_steps = num_epochs * len(train_dataloader)\n  lr_scheduler = get_scheduler(\n      \"linear\",\n      optimizer=optimizer,\n      num_warmup_steps=0,\n      num_training_steps=num_training_steps\n  )\n\n  progress_bar = tqdm(range(num_training_steps))\n\n  model.train()\n  for epoch in range(num_epochs):\n      for batch in train_dataloader:\n-         batch = {k: v.to(device) for k, v in batch.items()}\n          outputs = model(**batch)\n          loss = outputs.loss\n-         loss.backward()\n+         accelerator.backward(loss)\n\n          optimizer.step()\n          lr_scheduler.step()\n          optimizer.zero_grad()\n          progress_bar.update(1)\n```\n\n## Train\n\nOnce you've added the relevant lines of code, launch your training in a script or a notebook like Colaboratory.\n\n### Train with a script\n\nIf you are running your training from a script, run the following command to create and save a configuration file:\n\n```bash\naccelerate config\n```\n\nThen launch your training with:\n\n```bash\naccelerate launch train.py\n```\n\n### Train with a notebook\n\n\ud83e\udd17 Accelerate can also run in a notebook if you're planning on using Colaboratory's TPUs. Wrap all the code responsible for training in a function, and pass it to [`~accelerate.notebook_launcher`]:\n\n```py\n>>> from accelerate import notebook_launcher\n\n>>> notebook_launcher(training_function)\n```\n\nFor more information about \ud83e\udd17 Accelerate and its rich features, refer to the [documentation](https://huggingface.co/docs/accelerate)."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f0644d87-863c-4288-958b-2661314ec8c6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f0644d87-863c-4288-958b-2661314ec8c6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4fbae4eb-cf38-4d88-819d-a42b15b07a7f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5646aaa13f706a472cedfbc8fc5d1cb7e667bb5490d127cec146423143f317f6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2733, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "<!--\u26a0\ufe0f Note that this file is in Markdown but contains specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n-->\n\n# Community\n\nThis page regroups resources around \ud83e\udd17 Transformers developed by the community.\n\n## Community resources:\n\n| Resource     |      Description      |      Author      |\n|:----------|:-------------|------:|\n| [Hugging Face Transformers Glossary Flashcards](https://www.darigovresearch.com/huggingface-transformers-glossary-flashcards) | A set of flashcards based on the [Transformers Docs Glossary](glossary) that has been put into a form which can be easily learned/revised using [Anki](https://apps.ankiweb.net/) an open source, cross platform app specifically designed for long term knowledge retention. See this [Introductory video on how to use the flashcards](https://www.youtube.com/watch?v=Dji_h7PILrw). | [Darigov Research](https://www.darigovresearch.com/) |\n\n## Community notebooks:\n\n| Notebook     |      Description      |      Author      |      |\n|:----------|:-------------|:-------------|------:|\n| [Fine-tune a pre-trained Transformer to generate lyrics](https://github.com/AlekseyKorshuk/huggingartists) | How to generate lyrics in the style of your favorite artist by fine-tuning a GPT-2 model |  [Aleksey Korshuk](https://github.com/AlekseyKorshuk) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/AlekseyKorshuk/huggingartists/blob/master/huggingartists-demo.ipynb) |\n| [Train T5 in Tensorflow 2](https://github.com/snapthat/TF-T5-text-to-text) | How to train T5 for any task using Tensorflow 2. This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |\n| [Train T5 on TPU](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)  | How to train T5 on SQUAD with Transformers and Nlp | [Suraj Patil](https://github.com/patil-suraj) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil) |\n| [Fine-tune T5 for Classification and Multiple Choice](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb)  | How to fine-tune T5 for classification and multiple choice tasks using a text-to-text format with PyTorch Lightning |  [Suraj Patil](https://github.com/patil-suraj) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 4fbae4eb-cf38-4d88-819d-a42b15b07a7f
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"4fbae4eb-cf38-4d88-819d-a42b15b07a7f\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f0644d87-863c-4288-958b-2661314ec8c6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74757c3f59a27e93424cba24e98b9981afcfe44235c9057718462dbf88d7bba1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6a64ee0b-c895-4059-b874-b2419404cb13\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"03e4059e6cdb8d1f431fb51aa411f27bddccb5dc77a4ab65d7f877e7830585ac\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1669, \"end_char_idx\": 4102, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "This notebook demonstrates a Question & Answer task implemented in Tensorflow 2 using SQUAD | [Muhammad Harris](https://github.com/HarrisDePerceptron) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/snapthat/TF-T5-text-to-text/blob/master/snapthatT5/notebooks/TF-T5-Datasets%20Training.ipynb) |\n| [Train T5 on TPU](https://github.com/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb)  | How to train T5 on SQUAD with Transformers and Nlp | [Suraj Patil](https://github.com/patil-suraj) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/T5_on_TPU.ipynb#scrollTo=QLGiFCDqvuil) |\n| [Fine-tune T5 for Classification and Multiple Choice](https://github.com/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb)  | How to fine-tune T5 for classification and multiple choice tasks using a text-to-text format with PyTorch Lightning |  [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/exploring-T5/blob/master/t5_fine_tuning.ipynb) |\n| [Fine-tune DialoGPT on New Datasets and Languages](https://github.com/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb)  | How to fine-tune the DialoGPT model on a new dataset for open-dialog conversational chatbots |  [Nathan Cooper](https://github.com/ncoop57) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb) |\n| [Long Sequence Modeling with Reformer](https://github.com/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  | How to train on sequences as long as 500,000 tokens with Reformer |  [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  |\n| [Fine-tune BART for Summarization](https://github.com/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) | How to fine-tune BART for summarization with fastai using blurr | [Wayde Gilliam](https://ohmeow.com/) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 6a64ee0b-c895-4059-b874-b2419404cb13
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"6a64ee0b-c895-4059-b874-b2419404cb13\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4fbae4eb-cf38-4d88-819d-a42b15b07a7f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0cac1fd67084d1022359bc1214802498c70abc02bed0969e29a13c0f44654ea3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8c784227-2d51-46bf-a0dc-0bca9109dc32\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"43676eefa6ae10acac64ac2d310ae9b05be3d412612453e74c64b05b78b48f87\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3205, \"end_char_idx\": 5576, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ncoop57/i-am-a-nerd/blob/master/_notebooks/2020-05-12-chatbot-part-1.ipynb) |\n| [Long Sequence Modeling with Reformer](https://github.com/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  | How to train on sequences as long as 500,000 tokens with Reformer |  [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/PyTorch_Reformer.ipynb)  |\n| [Fine-tune BART for Summarization](https://github.com/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) | How to fine-tune BART for summarization with fastai using blurr | [Wayde Gilliam](https://ohmeow.com/) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ohmeow/ohmeow_website/blob/master/posts/2021-05-25-mbart-sequence-classification-with-blurr.ipynb) |\n| [Fine-tune a pre-trained Transformer on anyone's tweets](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) | How to generate tweets in the style of your favorite Twitter account by fine-tuning a GPT-2 model |  [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) |\n| [Optimize \ud83e\udd17 Hugging Face models with Weights & Biases](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) | A complete tutorial showcasing W&B integration with Hugging Face | [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) |\n| [Pretrain Longformer](https://github.com/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb)  | How to build a \"long\" version of existing pretrained models |  [Iz Beltagy](https://beltagy.net) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 8c784227-2d51-46bf-a0dc-0bca9109dc32
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8c784227-2d51-46bf-a0dc-0bca9109dc32\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6a64ee0b-c895-4059-b874-b2419404cb13\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d586fea41f6da6f0f3425c51bfce70fa736ac50c2babb49ec871734ac2f8d42d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3c23fa17-e7e1-4f20-b183-73c39b6a62a4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0d93a5a7e3890ef8f99723966e997099817e5d7fbcd3ef21d0a2968a08340ae8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4634, \"end_char_idx\": 6882, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/borisdayma/huggingtweets/blob/master/huggingtweets-demo.ipynb) |\n| [Optimize \ud83e\udd17 Hugging Face models with Weights & Biases](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) | A complete tutorial showcasing W&B integration with Hugging Face | [Boris Dayma](https://github.com/borisdayma) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_%26_Biases.ipynb) |\n| [Pretrain Longformer](https://github.com/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb)  | How to build a \"long\" version of existing pretrained models |  [Iz Beltagy](https://beltagy.net) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb) |\n| [Fine-tune Longformer for QA](https://github.com/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) | How to fine-tune longformer model for QA task | [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) |\n| [Evaluate Model with \ud83e\udd17nlp](https://github.com/patrickvonplaten/notebooks/blob/master/How_to_evaluate_Longformer_on_TriviaQA_using_NLP.ipynb) | How to evaluate longformer on TriviaQA with `nlp` | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1m7eTGlPmLRgoPkkA7rkhQdZ9ydpmsdLE?usp=sharing) |\n| [Fine-tune T5 for Sentiment Span Extraction](https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb)  | How to fine-tune T5 for sentiment span extraction using a text-to-text format with PyTorch Lightning |  [Lorenzo Ampil](https://github.com/enzoampil) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 3c23fa17-e7e1-4f20-b183-73c39b6a62a4
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3c23fa17-e7e1-4f20-b183-73c39b6a62a4\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8c784227-2d51-46bf-a0dc-0bca9109dc32\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"f42928e975e07076e62f5ece65bb8100d9c172af013b36309bafc4b47f7ad1e3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f7d1d5d3-dbe3-4e70-9af0-493eb17519b7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4f6876d0a4c0ebf1f9e8b456eeef0cbe1df29ed19164be9f0eda6dc283b2f8dc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5576, \"end_char_idx\": 7882, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/allenai/longformer/blob/master/scripts/convert_model_to_long.ipynb) |\n| [Fine-tune Longformer for QA](https://github.com/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) | How to fine-tune longformer model for QA task | [Suraj Patil](https://github.com/patil-suraj) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patil-suraj/Notebooks/blob/master/longformer_qa_training.ipynb) |\n| [Evaluate Model with \ud83e\udd17nlp](https://github.com/patrickvonplaten/notebooks/blob/master/How_to_evaluate_Longformer_on_TriviaQA_using_NLP.ipynb) | How to evaluate longformer on TriviaQA with `nlp` | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1m7eTGlPmLRgoPkkA7rkhQdZ9ydpmsdLE?usp=sharing) |\n| [Fine-tune T5 for Sentiment Span Extraction](https://github.com/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb)  | How to fine-tune T5 for sentiment span extraction using a text-to-text format with PyTorch Lightning |  [Lorenzo Ampil](https://github.com/enzoampil) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb) |\n| [Fine-tune DistilBert for Multiclass Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb) | How to fine-tune DistilBert for multiclass classification with PyTorch | [Abhishek Kumar Mishra](https://github.com/abhimishra91) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb)|\n|[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f7d1d5d3-dbe3-4e70-9af0-493eb17519b7
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f7d1d5d3-dbe3-4e70-9af0-493eb17519b7\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c23fa17-e7e1-4f20-b183-73c39b6a62a4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"a9e0f6716bcc42f4ae1e91ab58b7e2f4ddbe9b5219411ba5edf691d7b3c48f5e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"bf866754-cbbe-401e-bd1a-48f31cedfd46\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"907c9c21a581057ef6092e330668e407c94875911b4e0b8d8edfa29506e142a8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6882, \"end_char_idx\": 9359, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/enzoampil/t5-intro/blob/master/t5_qa_training_pytorch_span_extraction.ipynb) |\n| [Fine-tune DistilBert for Multiclass Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb) | How to fine-tune DistilBert for multiclass classification with PyTorch | [Abhishek Kumar Mishra](https://github.com/abhimishra91) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multiclass_classification.ipynb)|\n|[Fine-tune BERT for Multi-label Classification](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|How to fine-tune BERT for multi-label classification using PyTorch|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_multi_label_classification.ipynb)|\n|[Fine-tune T5 for Summarization](https://github.com/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|How to fine-tune T5 for summarization in PyTorch and track experiments with WandB|[Abhishek Kumar Mishra](https://github.com/abhimishra91) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|\n|[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|\n|[Pretrain Reformer for Masked Language Modeling](https://github.com/patrickvonplaten/notebooks/blob/master/Reformer_For_Masked_LM.ipynb)| How to train a Reformer model with bi-directional self-attention layers | [Patrick von Platen](https://github.com/patrickvonplaten) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: bf866754-cbbe-401e-bd1a-48f31cedfd46
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"bf866754-cbbe-401e-bd1a-48f31cedfd46\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f7d1d5d3-dbe3-4e70-9af0-493eb17519b7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b3ad9009bfdc68ae216debd2d5a68b1a9617ec528d54e4df93d55b343a656193\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0910f2a7-a57c-41dd-9e9a-5828b332a9e6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"85fa2245760f981b1f35660b7c49afa4abafb5fe617af081d9fb99ba38bfd74b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8377, \"end_char_idx\": 10803, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/abhimishra91/transformers-tutorials/blob/master/transformers_summarization_wandb.ipynb)|\n|[Speed up Fine-Tuning in Transformers with Dynamic Padding / Bucketing](https://github.com/ELS-RD/transformers-notebook/blob/master/Divide_Hugging_Face_Transformers_training_time_by_2_or_more.ipynb)|How to speed up fine-tuning by a factor of 2 using dynamic padding / bucketing|[Michael Benesty](https://github.com/pommedeterresautee) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1CBfRU1zbfu7-ijiOqAAQUA-RJaxfcJoO?usp=sharing)|\n|[Pretrain Reformer for Masked Language Modeling](https://github.com/patrickvonplaten/notebooks/blob/master/Reformer_For_Masked_LM.ipynb)| How to train a Reformer model with bi-directional self-attention layers | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1tzzh0i8PgDQGV3SMFUGxM7_gGae3K-uW?usp=sharing)|\n|[Expand and Fine Tune Sci-BERT](https://github.com/lordtt13/word-embeddings/blob/master/COVID-19%20Research%20Data/COVID-SciBERT.ipynb)| How to increase vocabulary of a pretrained SciBERT model from AllenAI on the CORD dataset and pipeline it. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1rqAR40goxbAfez1xvF3hBJphSCsvXmh8)|\n|[Fine Tune BlenderBotSmall for Summarization using the Trainer API](https://github.com/lordtt13/transformers-experiments/blob/master/Custom%20Tasks/fine-tune-blenderbot_small-for-summarization.ipynb)| How to fine-tune BlenderBotSmall for summarization on a custom dataset, using the Trainer API. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/19Wmupuls7mykSGyRN_Qo6lPQhgp56ymq?usp=sharing)|\n|[Fine-tune Electra and interpret with Integrated Gradients](https://github.com/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb) | How to fine-tune Electra for sentiment analysis and interpret predictions with Captum Integrated Gradients | [Eliza Szczechla](https://elsanns.github.io) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 0910f2a7-a57c-41dd-9e9a-5828b332a9e6
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"0910f2a7-a57c-41dd-9e9a-5828b332a9e6\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"bf866754-cbbe-401e-bd1a-48f31cedfd46\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"6ea5efbb16cf1500241c36b30d05cb03219f587baa76820d8752040ad23f957f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2d507b83-6251-46a2-a096-f50b676c038a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2bf0113b2a3c3b6e1ad1f33e5a921bb1700b307dfc6541bf459fe8dc67a0c88\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9767, \"end_char_idx\": 12337, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "| [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1rqAR40goxbAfez1xvF3hBJphSCsvXmh8)|\n|[Fine Tune BlenderBotSmall for Summarization using the Trainer API](https://github.com/lordtt13/transformers-experiments/blob/master/Custom%20Tasks/fine-tune-blenderbot_small-for-summarization.ipynb)| How to fine-tune BlenderBotSmall for summarization on a custom dataset, using the Trainer API. | [Tanmay Thakur](https://github.com/lordtt13) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/19Wmupuls7mykSGyRN_Qo6lPQhgp56ymq?usp=sharing)|\n|[Fine-tune Electra and interpret with Integrated Gradients](https://github.com/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb) | How to fine-tune Electra for sentiment analysis and interpret predictions with Captum Integrated Gradients | [Eliza Szczechla](https://elsanns.github.io) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/elsanns/xai-nlp-notebooks/blob/master/electra_fine_tune_interpret_captum_ig.ipynb)|\n|[fine-tune a non-English GPT-2 Model with Trainer class](https://github.com/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb) | How to fine-tune a non-English GPT-2 Model with Trainer class | [Philipp Schmid](https://www.philschmid.de) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb)|\n|[Fine-tune a DistilBERT Model for Multi Label Classification task](https://github.com/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb) | How to fine-tune a DistilBERT Model for Multi Label Classification task | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb)|\n|[Fine-tune ALBERT for sentence-pair classification](https://github.com/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb) | How to fine-tune an ALBERT model or another BERT-based model for the sentence-pair classification task | [Nadir El Manouzi](https://github.com/NadirEM) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 2d507b83-6251-46a2-a096-f50b676c038a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"2d507b83-6251-46a2-a096-f50b676c038a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0910f2a7-a57c-41dd-9e9a-5828b332a9e6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"48847e1090dbe7503e4bbe3262eb3c518c13ec6378b6668a0c109e4d29f78ae3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"31df94ea-8005-4005-a0d2-795359ec7851\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"81394bed4305e0d14d84fad18b41452ea7a31658c6b5493e6766e1f856ab11eb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11293, \"end_char_idx\": 13629, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/philschmid/fine-tune-GPT-2/blob/master/Fine_tune_a_non_English_GPT_2_Model_with_Huggingface.ipynb)|\n|[Fine-tune a DistilBERT Model for Multi Label Classification task](https://github.com/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb) | How to fine-tune a DistilBERT Model for Multi Label Classification task | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/Transformers_scripts/blob/master/Transformers_multilabel_distilbert.ipynb)|\n|[Fine-tune ALBERT for sentence-pair classification](https://github.com/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb) | How to fine-tune an ALBERT model or another BERT-based model for the sentence-pair classification task | [Nadir El Manouzi](https://github.com/NadirEM) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb)|\n|[Fine-tune Roberta for sentiment analysis](https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb) | How to fine-tune a Roberta model for sentiment analysis | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb)|\n|[Evaluating Question Generation Models](https://github.com/flexudy-pipe/qugeev) | How accurate are the answers to questions generated by your seq2seq transformer model? | [Pascal Zoleko](https://github.com/zolekode) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bpsSqCQU-iw_5nNoRm_crPq6FRuJthq_?usp=sharing)|\n|[Classify text with DistilBERT and Tensorflow](https://github.com/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb) | How to fine-tune DistilBERT for text classification in TensorFlow | [Peter Bayerle](https://github.com/peterbayerle) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 31df94ea-8005-4005-a0d2-795359ec7851
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"31df94ea-8005-4005-a0d2-795359ec7851\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2d507b83-6251-46a2-a096-f50b676c038a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"55cb64c04f0c9143458b64aeaa0ac09c338dfc9e1301597619a59d98cf97f83a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"960b669f-1883-4a91-80b4-54b862db868b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4d1b70bf401d7b8d1835e95f85b9c6c7c7c3a5c66cce3eefc058e29e42fc239c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12337, \"end_char_idx\": 14703, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NadirEM/nlp-notebooks/blob/master/Fine_tune_ALBERT_sentence_pair_classification.ipynb)|\n|[Fine-tune Roberta for sentiment analysis](https://github.com/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb) | How to fine-tune a Roberta model for sentiment analysis | [Dhaval Taunk](https://github.com/DhavalTaunk08) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/DhavalTaunk08/NLP_scripts/blob/master/sentiment_analysis_using_roberta.ipynb)|\n|[Evaluating Question Generation Models](https://github.com/flexudy-pipe/qugeev) | How accurate are the answers to questions generated by your seq2seq transformer model? | [Pascal Zoleko](https://github.com/zolekode) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1bpsSqCQU-iw_5nNoRm_crPq6FRuJthq_?usp=sharing)|\n|[Classify text with DistilBERT and Tensorflow](https://github.com/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb) | How to fine-tune DistilBERT for text classification in TensorFlow | [Peter Bayerle](https://github.com/peterbayerle) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb)|\n|[Leverage BERT for Encoder-Decoder Summarization on CNN/Dailymail](https://github.com/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb) | How to warm-start a *EncoderDecoderModel* with a *google-bert/bert-base-uncased* checkpoint for summarization on CNN/Dailymail | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb)|\n|[Leverage RoBERTa for Encoder-Decoder Summarization on BBC XSum](https://github.com/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb) | How to warm-start a shared *EncoderDecoderModel* with a *FacebookAI/roberta-base* checkpoint for summarization on BBC/XSum | [Patrick von Platen](https://github.com/patrickvonplaten) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 960b669f-1883-4a91-80b4-54b862db868b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"960b669f-1883-4a91-80b4-54b862db868b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"31df94ea-8005-4005-a0d2-795359ec7851\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"0c3e7de539006203d02e2e493c671c0fc62eda424de1330a4453386a848b97be\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8e09dbce-ad13-4bde-a539-8a09637ef9db\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9d8ffb1ae73fff6d8f6498f4c12d81e2909193bda9968544980adea14993260b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13629, \"end_char_idx\": 15876, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/peterbayerle/huggingface_notebook/blob/main/distilbert_tf.ipynb)|\n|[Leverage BERT for Encoder-Decoder Summarization on CNN/Dailymail](https://github.com/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb) | How to warm-start a *EncoderDecoderModel* with a *google-bert/bert-base-uncased* checkpoint for summarization on CNN/Dailymail | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/BERT2BERT_for_CNN_Dailymail.ipynb)|\n|[Leverage RoBERTa for Encoder-Decoder Summarization on BBC XSum](https://github.com/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb) | How to warm-start a shared *EncoderDecoderModel* with a *FacebookAI/roberta-base* checkpoint for summarization on BBC/XSum | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb)|\n|[Fine-tune TAPAS on Sequential Question Answering (SQA)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb) | How to fine-tune *TapasForQuestionAnswering* with a *tapas-base* checkpoint on the Sequential Question Answering (SQA) dataset | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb)|\n|[Evaluate TAPAS on Table Fact Checking (TabFact)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb) | How to evaluate a fine-tuned *TapasForSequenceClassification* with a *tapas-base-finetuned-tabfact* checkpoint using a combination of the \ud83e\udd17 datasets and \ud83e\udd17 transformers libraries | [Niels Rogge](https://github.com/nielsrogge) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 8e09dbce-ad13-4bde-a539-8a09637ef9db
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"8e09dbce-ad13-4bde-a539-8a09637ef9db\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"960b669f-1883-4a91-80b4-54b862db868b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9147ae27054d6af462ef8b943d5525823b7388d22adb2a75db88f5c8d36c85c1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d7cca092-eb4a-4fce-aff6-06a31c6ce477\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2ac6a16f6b7109db2f03ad1df0552e6370961d89eee34ed421acfcb26efbd4d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14703, \"end_char_idx\": 17436, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/RoBERTaShared_for_BBC_XSum.ipynb)|\n|[Fine-tune TAPAS on Sequential Question Answering (SQA)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb) | How to fine-tune *TapasForQuestionAnswering* with a *tapas-base* checkpoint on the Sequential Question Answering (SQA) dataset | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Fine_tuning_TapasForQuestionAnswering_on_SQA.ipynb)|\n|[Evaluate TAPAS on Table Fact Checking (TabFact)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb) | How to evaluate a fine-tuned *TapasForSequenceClassification* with a *tapas-base-finetuned-tabfact* checkpoint using a combination of the \ud83e\udd17 datasets and \ud83e\udd17 transformers libraries | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/TAPAS/Evaluating_TAPAS_on_the_Tabfact_test_set.ipynb)|\n|[Fine-tuning mBART for translation](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb) | How to fine-tune mBART using Seq2SeqTrainer for Hindi to English translation | [Vasudev Gupta](https://github.com/vasudevgupta7) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb)|\n|[Fine-tune LayoutLM on FUNSD (a form understanding dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb) | How to fine-tune *LayoutLMForTokenClassification* on the FUNSD dataset for information extraction from scanned documents | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb)|\n|[Fine-Tune DistilGPT2 and Generate Text](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb) | How to fine-tune DistilGPT2 and generate text | [Aakash Tripathi](https://github.com/tripathiaakash) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: d7cca092-eb4a-4fce-aff6-06a31c6ce477
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d7cca092-eb4a-4fce-aff6-06a31c6ce477\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8e09dbce-ad13-4bde-a539-8a09637ef9db\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e7a4a68c89db29749dfa854d37ea870f0735b81b95ca660d59332132e94d5e4f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f722dc3e-1dbf-45d5-af8e-7fd623f74065\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5ad18b33ca524874f40ea2c8c82314066533866bf20837faf9555c311f9123ac\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16380, \"end_char_idx\": 18946, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vasudevgupta7/huggingface-tutorials/blob/main/translation_training.ipynb)|\n|[Fine-tune LayoutLM on FUNSD (a form understanding dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb) | How to fine-tune *LayoutLMForTokenClassification* on the FUNSD dataset for information extraction from scanned documents | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForTokenClassification_on_FUNSD.ipynb)|\n|[Fine-Tune DistilGPT2 and Generate Text](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb) | How to fine-tune DistilGPT2 and generate text | [Aakash Tripathi](https://github.com/tripathiaakash) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/tripathiaakash/DistilGPT2-Tutorial/blob/main/distilgpt2_fine_tuning.ipynb)|\n|[Fine-Tune LED on up to 8K tokens](https://github.com/patrickvonplaten/notebooks/blob/master/Fine_tune_Longformer_Encoder_Decoder_(LED)_for_Summarization_on_pubmed.ipynb) | How to fine-tune LED on pubmed for long-range summarization | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_tune_Longformer_Encoder_Decoder_(LED)_for_Summarization_on_pubmed.ipynb)|\n|[Evaluate LED on Arxiv](https://github.com/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb) | How to effectively evaluate LED on long-range summarization | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb)|\n|[Fine-tune LayoutLM on RVL-CDIP (a document image classification dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForSequenceClassification_on_RVL_CDIP.ipynb) | How to fine-tune *LayoutLMForSequenceClassification* on the RVL-CDIP dataset for scanned document classification | [Niels Rogge](https://github.com/nielsrogge) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: f722dc3e-1dbf-45d5-af8e-7fd623f74065
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"f722dc3e-1dbf-45d5-af8e-7fd623f74065\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d7cca092-eb4a-4fce-aff6-06a31c6ce477\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"241de393eaa45675b35b5331e7af6d912404ddc32a7343e02f582412f5989df5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"71d9d6b5-6ce2-49fa-9764-50dfb7ce1c8a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"efbb1bef0f0869670a2cc0b02a62fbbf397b2293d8f75852a29d8bd8fd617634\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17926, \"end_char_idx\": 20347, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Fine_tune_Longformer_Encoder_Decoder_(LED)_for_Summarization_on_pubmed.ipynb)|\n|[Evaluate LED on Arxiv](https://github.com/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb) | How to effectively evaluate LED on long-range summarization | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/LED_on_Arxiv.ipynb)|\n|[Fine-tune LayoutLM on RVL-CDIP (a document image classification dataset)](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForSequenceClassification_on_RVL_CDIP.ipynb) | How to fine-tune *LayoutLMForSequenceClassification* on the RVL-CDIP dataset for scanned document classification | [Niels Rogge](https://github.com/nielsrogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/LayoutLM/Fine_tuning_LayoutLMForSequenceClassification_on_RVL_CDIP.ipynb)|\n|[Wav2Vec2 CTC decoding with GPT2 adjustment](https://github.com/voidful/huggingface_notebook/blob/main/xlsr_gpt.ipynb) | How to decode CTC sequence with language model adjustment | [Eric Lam](https://github.com/voidful) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1e_z5jQHYbO2YKEaUgzb1ww1WwiAyydAj?usp=sharing)|\n|[Fine-tune BART for summarization in two languages with Trainer class](https://github.com/elsanns/xai-nlp-notebooks/blob/master/fine_tune_bart_summarization_two_langs.ipynb) | How to fine-tune BART for summarization in two languages with Trainer class | [Eliza Szczechla](https://github.com/elsanns) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/elsanns/xai-nlp-notebooks/blob/master/fine_tune_bart_summarization_two_langs.ipynb)|\n|[Evaluate Big Bird on Trivia QA](https://github.com/patrickvonplaten/notebooks/blob/master/Evaluating_Big_Bird_on_TriviaQA.ipynb) | How to evaluate BigBird on long document question answering on Trivia QA | [Patrick von Platen](https://github.com/patrickvonplaten) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 71d9d6b5-6ce2-49fa-9764-50dfb7ce1c8a
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"71d9d6b5-6ce2-49fa-9764-50dfb7ce1c8a\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f722dc3e-1dbf-45d5-af8e-7fd623f74065\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"af040384843084370ddeebeeb60e708a758ad5b169cffa2b7a7bbf15977d7577\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a4cafa83-47d7-45ba-8055-c6444c3a2421\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c334f9b6cc26964de64e7a5852add96f0ec3bb6db3ac6237a1751b9b85d77e9d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19408, \"end_char_idx\": 21397, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1e_z5jQHYbO2YKEaUgzb1ww1WwiAyydAj?usp=sharing)|\n|[Fine-tune BART for summarization in two languages with Trainer class](https://github.com/elsanns/xai-nlp-notebooks/blob/master/fine_tune_bart_summarization_two_langs.ipynb) | How to fine-tune BART for summarization in two languages with Trainer class | [Eliza Szczechla](https://github.com/elsanns) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/elsanns/xai-nlp-notebooks/blob/master/fine_tune_bart_summarization_two_langs.ipynb)|\n|[Evaluate Big Bird on Trivia QA](https://github.com/patrickvonplaten/notebooks/blob/master/Evaluating_Big_Bird_on_TriviaQA.ipynb) | How to evaluate BigBird on long document question answering on Trivia QA | [Patrick von Platen](https://github.com/patrickvonplaten) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Evaluating_Big_Bird_on_TriviaQA.ipynb)|\n| [Create video captions using Wav2Vec2](https://github.com/Muennighoff/ytclipcc/blob/main/wav2vec_youtube_captions.ipynb) | How to create YouTube captions from any video by transcribing the audio with Wav2Vec | [Niklas Muennighoff](https://github.com/Muennighoff) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Muennighoff/ytclipcc/blob/main/wav2vec_youtube_captions.ipynb) |\n| [Fine-tune the Vision Transformer on CIFAR-10 using PyTorch Lightning](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_PyTorch_Lightning.ipynb) | How to fine-tune the Vision Transformer (ViT) on CIFAR-10 using HuggingFace Transformers, Datasets and PyTorch Lightning | [Niels Rogge](https://github.com/nielsrogge) |[!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: a4cafa83-47d7-45ba-8055-c6444c3a2421
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"a4cafa83-47d7-45ba-8055-c6444c3a2421\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"71d9d6b5-6ce2-49fa-9764-50dfb7ce1c8a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"5f5ab0b4cb8a4d6c3a629d2222f5b8bf136fe8b7c3ac25d5f7b2a33acfbe7997\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a4f746dc-c689-441c-8fe0-7e5953ae3173\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b49382e8ec3c496a6546d7ce7234f59ab3ed0a0d2e6de36c71c6c98eff613089\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20347, \"end_char_idx\": 22601, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/patrickvonplaten/notebooks/blob/master/Evaluating_Big_Bird_on_TriviaQA.ipynb)|\n| [Create video captions using Wav2Vec2](https://github.com/Muennighoff/ytclipcc/blob/main/wav2vec_youtube_captions.ipynb) | How to create YouTube captions from any video by transcribing the audio with Wav2Vec | [Niklas Muennighoff](https://github.com/Muennighoff) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Muennighoff/ytclipcc/blob/main/wav2vec_youtube_captions.ipynb) |\n| [Fine-tune the Vision Transformer on CIFAR-10 using PyTorch Lightning](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_PyTorch_Lightning.ipynb) | How to fine-tune the Vision Transformer (ViT) on CIFAR-10 using HuggingFace Transformers, Datasets and PyTorch Lightning | [Niels Rogge](https://github.com/nielsrogge) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_PyTorch_Lightning.ipynb) |\n| [Fine-tune the Vision Transformer on CIFAR-10 using the \ud83e\udd17 Trainer](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_the_%F0%9F%A4%97_Trainer.ipynb) | How to fine-tune the Vision Transformer (ViT) on CIFAR-10 using HuggingFace Transformers, Datasets and the \ud83e\udd17 Trainer | [Niels Rogge](https://github.com/nielsrogge) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_the_%F0%9F%A4%97_Trainer.ipynb) |\n| [Evaluate LUKE on Open Entity, an entity typing dataset](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_open_entity.ipynb) | How to evaluate *LukeForEntityClassification* on the Open Entity dataset | [Ikuya Yamada](https://github.com/ikuyamada) |[!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: a4f746dc-c689-441c-8fe0-7e5953ae3173
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"a4f746dc-c689-441c-8fe0-7e5953ae3173\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a4cafa83-47d7-45ba-8055-c6444c3a2421\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"e6adedf0de056165e868a1ea43a9cee769668d353388004a864f0c9c1efa08af\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5a4dd43d-102f-40a4-a999-dff5f5986ce1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d354221b9e7c8ee88d8535fdd44dc929672beac5dfa0df074ebd02aa949684f1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21397, \"end_char_idx\": 23982, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_PyTorch_Lightning.ipynb) |\n| [Fine-tune the Vision Transformer on CIFAR-10 using the \ud83e\udd17 Trainer](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_the_%F0%9F%A4%97_Trainer.ipynb) | How to fine-tune the Vision Transformer (ViT) on CIFAR-10 using HuggingFace Transformers, Datasets and the \ud83e\udd17 Trainer | [Niels Rogge](https://github.com/nielsrogge) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/VisionTransformer/Fine_tuning_the_Vision_Transformer_on_CIFAR_10_with_the_%F0%9F%A4%97_Trainer.ipynb) |\n| [Evaluate LUKE on Open Entity, an entity typing dataset](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_open_entity.ipynb) | How to evaluate *LukeForEntityClassification* on the Open Entity dataset | [Ikuya Yamada](https://github.com/ikuyamada) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_open_entity.ipynb) |\n| [Evaluate LUKE on TACRED, a relation extraction dataset](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_tacred.ipynb) | How to evaluate *LukeForEntityPairClassification* on the TACRED dataset | [Ikuya Yamada](https://github.com/ikuyamada) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_tacred.ipynb) |\n| [Evaluate LUKE on CoNLL-2003, an important NER benchmark](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_conll_2003.ipynb) | How to evaluate *LukeForEntitySpanClassification* on the CoNLL-2003 dataset | [Ikuya Yamada](https://github.com/ikuyamada) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_conll_2003.ipynb) |\n| [Evaluate BigBird-Pegasus on PubMed dataset](https://github.com/vasudevgupta7/bigbird/blob/main/notebooks/bigbird_pegasus_evaluation.ipynb) | How to evaluate *BigBirdPegasusForConditionalGeneration* on PubMed dataset | [Vasudev Gupta](https://github.com/vasudevgupta7) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 5a4dd43d-102f-40a4-a999-dff5f5986ce1
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5a4dd43d-102f-40a4-a999-dff5f5986ce1\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a4f746dc-c689-441c-8fe0-7e5953ae3173\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"9930f64b672dee2107d148977b687f8e97b83fbf0abd7f6055a90a4bb1af667b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c4da2d45-1416-4c03-8cce-5e375b8584ab\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f7d658bf782333a73641622c708d7a5f434e7253ee90567e028039a46c0ffe1f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23058, \"end_char_idx\": 25572, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_tacred.ipynb) |\n| [Evaluate LUKE on CoNLL-2003, an important NER benchmark](https://github.com/studio-ousia/luke/blob/master/notebooks/huggingface_conll_2003.ipynb) | How to evaluate *LukeForEntitySpanClassification* on the CoNLL-2003 dataset | [Ikuya Yamada](https://github.com/ikuyamada) |[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/studio-ousia/luke/blob/master/notebooks/huggingface_conll_2003.ipynb) |\n| [Evaluate BigBird-Pegasus on PubMed dataset](https://github.com/vasudevgupta7/bigbird/blob/main/notebooks/bigbird_pegasus_evaluation.ipynb) | How to evaluate *BigBirdPegasusForConditionalGeneration* on PubMed dataset | [Vasudev Gupta](https://github.com/vasudevgupta7) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vasudevgupta7/bigbird/blob/main/notebooks/bigbird_pegasus_evaluation.ipynb) |\n| [Speech Emotion Classification with Wav2Vec2](https://github/m3hrdadfi/soxan/blob/main/notebooks/Emotion_recognition_in_Greek_speech_using_Wav2Vec2.ipynb) | How to leverage a pretrained Wav2Vec2 model for Emotion Classification on the MEGA dataset | [Mehrdad Farahani](https://github.com/m3hrdadfi) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/m3hrdadfi/soxan/blob/main/notebooks/Emotion_recognition_in_Greek_speech_using_Wav2Vec2.ipynb) |\n| [Detect objects in an image with DETR](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/DETR/DETR_minimal_example_(with_DetrFeatureExtractor).ipynb) | How to use a trained *DetrForObjectDetection* model to detect objects in an image and visualize attention | [Niels Rogge](https://github.com/NielsRogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/DETR/DETR_minimal_example_(with_DetrFeatureExtractor).ipynb) |\n| [Fine-tune DETR on a custom object detection dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/DETR/Fine_tuning_DetrForObjectDetection_on_custom_dataset_(balloon).ipynb) | How to fine-tune *DetrForObjectDetection* on a custom object detection dataset | [Niels Rogge](https://github.com/NielsRogge) | [!"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: c4da2d45-1416-4c03-8cce-5e375b8584ab
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/community", "title": "Community", "tokens": 7568, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"c4da2d45-1416-4c03-8cce-5e375b8584ab\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"98a81be9-b09f-4d62-b899-6ed78b4ebb8a\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"74d714af8e35fb1703fcc4f44e23228f715556eae56a6c7cf14ac422368d5a08\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5a4dd43d-102f-40a4-a999-dff5f5986ce1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/community\", \"title\": \"Community\", \"tokens\": 7568, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"042ce2a793b648ceb4dd9b88416732f08900ec361f6032c5d461bced7f02cb8f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24481, \"end_char_idx\": 26213, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "ref_doc_id": "98a81be9-b09f-4d62-b899-6ed78b4ebb8a", "chroma:document": "[Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/m3hrdadfi/soxan/blob/main/notebooks/Emotion_recognition_in_Greek_speech_using_Wav2Vec2.ipynb) |\n| [Detect objects in an image with DETR](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/DETR/DETR_minimal_example_(with_DetrFeatureExtractor).ipynb) | How to use a trained *DetrForObjectDetection* model to detect objects in an image and visualize attention | [Niels Rogge](https://github.com/NielsRogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/DETR/DETR_minimal_example_(with_DetrFeatureExtractor).ipynb) |\n| [Fine-tune DETR on a custom object detection dataset](https://github.com/NielsRogge/Transformers-Tutorials/blob/master/DETR/Fine_tuning_DetrForObjectDetection_on_custom_dataset_(balloon).ipynb) | How to fine-tune *DetrForObjectDetection* on a custom object detection dataset | [Niels Rogge](https://github.com/NielsRogge) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/DETR/Fine_tuning_DetrForObjectDetection_on_custom_dataset_(balloon).ipynb) |\n| [Finetune T5 for Named Entity Recognition](https://github.com/ToluClassics/Notebooks/blob/main/T5_Ner_Finetuning.ipynb) | How to fine-tune *T5* on a Named Entity Recognition Task | [Ogundepo Odunayo](https://github.com/ToluClassics) | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1obr78FY_cBmWY5ODViCmzdY6O1KB65Vc?usp=sharing) |"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: cd44dd36-5357-4c6c-b068-701da95e390b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/troubleshooting", "title": "Troubleshoot", "tokens": 2128, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"cd44dd36-5357-4c6c-b068-701da95e390b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f574da76-3507-4737-9ba1-f03f993b1b06\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cf7ceb63fcd69053af31c31e769d7cbe115106f05438ffa92b3abb285b66d3aa\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3250db8f-5e23-496e-83d8-dff8a2446b21\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b3dd10e4f76c6551c63850c81e5b0af2754fd19389c8c589fd05bb452412f72f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3294, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "ref_doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "chroma:document": "<!---\nCopyright 2022 The HuggingFace Team. All rights reserved.\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n\n\u26a0\ufe0f Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be\nrendered properly in your Markdown viewer.\n\n-->\n\n# Troubleshoot\n\nSometimes errors occur, but we are here to help! This guide covers some of the most common issues we've seen and how you can resolve them. However, this guide isn't meant to be a comprehensive collection of every \ud83e\udd17 Transformers issue. For more help with troubleshooting your issue, try:\n\n<Youtube id=\"S2EEG3JIt2A\"/>\n\n1. Asking for help on the [forums](https://discuss.huggingface.co/). There are specific categories you can post your question to, like [Beginners](https://discuss.huggingface.co/c/beginners/5) or [\ud83e\udd17 Transformers](https://discuss.huggingface.co/c/transformers/9). Make sure you write a good descriptive forum post with some reproducible code to maximize the likelihood that your problem is solved!\n\n<Youtube id=\"_PAli-V4wj0\"/>\n\n2. Create an [Issue](https://github.com/huggingface/transformers/issues/new/choose) on the \ud83e\udd17 Transformers repository if it is a bug related to the library. Try to include as much information describing the bug as possible to help us better figure out what's wrong and how we can fix it.\n\n3. Check the [Migration](migration) guide if you use an older version of \ud83e\udd17 Transformers since some important changes have been introduced between versions.\n\nFor more details about troubleshooting and getting help, take a look at [Chapter 8](https://huggingface.co/course/chapter8/1?fw=pt) of the Hugging Face course.\n\n\n## Firewalled environments\n\nSome GPU instances on cloud and intranet setups are firewalled to external connections, resulting in a connection error. When your script attempts to download model weights or datasets, the download will hang and then timeout with the following message:\n\n```\nValueError: Connection error, and we cannot find the requested files in the cached path.\nPlease try again or make sure your Internet connection is on.\n```\n\nIn this case, you should try to run \ud83e\udd17 Transformers on [offline mode](installation#offline-mode) to avoid the connection error.\n\n## CUDA out of memory\n\nTraining large models with millions of parameters can be challenging without the appropriate hardware. A common error you may encounter when the GPU runs out of memory is:\n\n```\nCUDA out of memory. Tried to allocate 256.00 MiB (GPU 0; 11.17 GiB total capacity; 9.70 GiB already allocated; 179.81 MiB free; 9.85 GiB reserved in total by PyTorch)\n```\n\nHere are some potential solutions you can try to lessen memory use:\n\n- Reduce the [`per_device_train_batch_size`](main_classes/trainer#transformers.TrainingArguments.per_device_train_batch_size) value in [`TrainingArguments`]."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 3250db8f-5e23-496e-83d8-dff8a2446b21
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/troubleshooting", "title": "Troubleshoot", "tokens": 2128, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"3250db8f-5e23-496e-83d8-dff8a2446b21\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f574da76-3507-4737-9ba1-f03f993b1b06\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cf7ceb63fcd69053af31c31e769d7cbe115106f05438ffa92b3abb285b66d3aa\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cd44dd36-5357-4c6c-b068-701da95e390b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1da95ab0384de49a8832d67734cb621930471eb74752c7e9d7a3602fb0a7191a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d90aa1b9-f528-4e93-9b9c-bcbc311035cb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"568b298d42e80db4371c2dcff4595575dee08541b8363fab802dafe31b46e4f9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2125, \"end_char_idx\": 5499, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "ref_doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "chroma:document": "## Firewalled environments\n\nSome GPU instances on cloud and intranet setups are firewalled to external connections, resulting in a connection error. When your script attempts to download model weights or datasets, the download will hang and then timeout with the following message:\n\n```\nValueError: Connection error, and we cannot find the requested files in the cached path.\nPlease try again or make sure your Internet connection is on.\n```\n\nIn this case, you should try to run \ud83e\udd17 Transformers on [offline mode](installation#offline-mode) to avoid the connection error.\n\n## CUDA out of memory\n\nTraining large models with millions of parameters can be challenging without the appropriate hardware. A common error you may encounter when the GPU runs out of memory is:\n\n```\nCUDA out of memory. Tried to allocate 256.00 MiB (GPU 0; 11.17 GiB total capacity; 9.70 GiB already allocated; 179.81 MiB free; 9.85 GiB reserved in total by PyTorch)\n```\n\nHere are some potential solutions you can try to lessen memory use:\n\n- Reduce the [`per_device_train_batch_size`](main_classes/trainer#transformers.TrainingArguments.per_device_train_batch_size) value in [`TrainingArguments`].\n- Try using [`gradient_accumulation_steps`](main_classes/trainer#transformers.TrainingArguments.gradient_accumulation_steps) in [`TrainingArguments`] to effectively increase overall batch size.\n\n<Tip>\n\nRefer to the Performance [guide](performance) for more details about memory-saving techniques.\n\n</Tip>\n\n## Unable to load a saved TensorFlow model\n\nTensorFlow's [model.save](https://www.tensorflow.org/tutorials/keras/save_and_load#save_the_entire_model) method will save the entire model - architecture, weights, training configuration - in a single file. However, when you load the model file again, you may run into an error because \ud83e\udd17 Transformers may not load all the TensorFlow-related objects in the model file. To avoid issues with saving and loading TensorFlow models, we recommend you:\n\n- Save the model weights as a `h5` file extension with [`model.save_weights`](https://www.tensorflow.org/tutorials/keras/save_and_load#save_the_entire_model) and then reload the model with [`~TFPreTrainedModel.from_pretrained`]:\n\n```py\n>>> from transformers import TFPreTrainedModel\n>>> from tensorflow import keras\n\n>>> model.save_weights(\"some_folder/tf_model.h5\")\n>>> model = TFPreTrainedModel.from_pretrained(\"some_folder\")\n```\n\n- Save the model with [`~TFPretrainedModel.save_pretrained`] and load it again with [`~TFPreTrainedModel.from_pretrained`]:\n\n```py\n>>> from transformers import TFPreTrainedModel\n\n>>> model.save_pretrained(\"path_to/model\")\n>>> model = TFPreTrainedModel.from_pretrained(\"path_to/model\")\n```\n\n## ImportError\n\nAnother common error you may encounter, especially if it is a newly released model, is `ImportError`:\n\n```\nImportError: cannot import name 'ImageGPTImageProcessor' from 'transformers' (unknown location)\n```\n\nFor these error types, check to make sure you have the latest version of \ud83e\udd17 Transformers installed to access the most recent models:\n\n```bash\npip install transformers --upgrade\n```\n\n## CUDA error: device-side assert triggered\n\nSometimes you may run into a generic CUDA error about an error in the device code.\n\n```\nRuntimeError: CUDA error: device-side assert triggered\n```\n\nYou should try to run the code on a CPU first to get a more descriptive error message."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: d90aa1b9-f528-4e93-9b9c-bcbc311035cb
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/troubleshooting", "title": "Troubleshoot", "tokens": 2128, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"d90aa1b9-f528-4e93-9b9c-bcbc311035cb\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f574da76-3507-4737-9ba1-f03f993b1b06\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cf7ceb63fcd69053af31c31e769d7cbe115106f05438ffa92b3abb285b66d3aa\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3250db8f-5e23-496e-83d8-dff8a2446b21\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"7e813b2db2bdca09ca22968e591bdff8ebc326d78dfdcae1f448e72b12dbf2e7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6854394c-ac5f-4f21-940d-feca2c3310ac\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d092617d771890efe17cf01a85b00b48d02a30f3f9a2f5934b0f548cb950fac5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3853, \"end_char_idx\": 6260, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "ref_doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "chroma:document": "However, when you load the model file again, you may run into an error because \ud83e\udd17 Transformers may not load all the TensorFlow-related objects in the model file. To avoid issues with saving and loading TensorFlow models, we recommend you:\n\n- Save the model weights as a `h5` file extension with [`model.save_weights`](https://www.tensorflow.org/tutorials/keras/save_and_load#save_the_entire_model) and then reload the model with [`~TFPreTrainedModel.from_pretrained`]:\n\n```py\n>>> from transformers import TFPreTrainedModel\n>>> from tensorflow import keras\n\n>>> model.save_weights(\"some_folder/tf_model.h5\")\n>>> model = TFPreTrainedModel.from_pretrained(\"some_folder\")\n```\n\n- Save the model with [`~TFPretrainedModel.save_pretrained`] and load it again with [`~TFPreTrainedModel.from_pretrained`]:\n\n```py\n>>> from transformers import TFPreTrainedModel\n\n>>> model.save_pretrained(\"path_to/model\")\n>>> model = TFPreTrainedModel.from_pretrained(\"path_to/model\")\n```\n\n## ImportError\n\nAnother common error you may encounter, especially if it is a newly released model, is `ImportError`:\n\n```\nImportError: cannot import name 'ImageGPTImageProcessor' from 'transformers' (unknown location)\n```\n\nFor these error types, check to make sure you have the latest version of \ud83e\udd17 Transformers installed to access the most recent models:\n\n```bash\npip install transformers --upgrade\n```\n\n## CUDA error: device-side assert triggered\n\nSometimes you may run into a generic CUDA error about an error in the device code.\n\n```\nRuntimeError: CUDA error: device-side assert triggered\n```\n\nYou should try to run the code on a CPU first to get a more descriptive error message. Add the following environment variable to the beginning of your code to switch to a CPU:\n\n```py\n>>> import os\n\n>>> os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n```\n\nAnother option is to get a better traceback from the GPU. Add the following environment variable to the beginning of your code to get the traceback to point to the source of the error:\n\n```py\n>>> import os\n\n>>> os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n```\n\n## Incorrect output when padding tokens aren't masked\n\nIn some cases, the output `hidden_state` may be incorrect if the `input_ids` include padding tokens. To demonstrate, load a model and tokenizer. You can access a model's `pad_token_id` to see its value. The `pad_token_id` may be `None` for some models, but you can always manually set it."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 6854394c-ac5f-4f21-940d-feca2c3310ac
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/troubleshooting", "title": "Troubleshoot", "tokens": 2128, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"6854394c-ac5f-4f21-940d-feca2c3310ac\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f574da76-3507-4737-9ba1-f03f993b1b06\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cf7ceb63fcd69053af31c31e769d7cbe115106f05438ffa92b3abb285b66d3aa\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d90aa1b9-f528-4e93-9b9c-bcbc311035cb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"d949a44b8448370e7fe0b8f1778455d9ac4a2c1fa78b4debd4098eaddb920fdc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9c91336f-b6ba-4a00-b1ca-3a9c9ffc7c6b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"600f469dd4f389d108283ff543bffeac5b0baa6e042a33481ac5febbdedadcaf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5349, \"end_char_idx\": 8395, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "ref_doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "chroma:document": "```\nRuntimeError: CUDA error: device-side assert triggered\n```\n\nYou should try to run the code on a CPU first to get a more descriptive error message. Add the following environment variable to the beginning of your code to switch to a CPU:\n\n```py\n>>> import os\n\n>>> os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n```\n\nAnother option is to get a better traceback from the GPU. Add the following environment variable to the beginning of your code to get the traceback to point to the source of the error:\n\n```py\n>>> import os\n\n>>> os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n```\n\n## Incorrect output when padding tokens aren't masked\n\nIn some cases, the output `hidden_state` may be incorrect if the `input_ids` include padding tokens. To demonstrate, load a model and tokenizer. You can access a model's `pad_token_id` to see its value. The `pad_token_id` may be `None` for some models, but you can always manually set it.\n\n```py\n>>> from transformers import AutoModelForSequenceClassification\n>>> import torch\n\n>>> model = AutoModelForSequenceClassification.from_pretrained(\"google-bert/bert-base-uncased\")\n>>> model.config.pad_token_id\n0\n```\n\nThe following example shows the output without masking the padding tokens:\n\n```py\n>>> input_ids = torch.tensor([[7592, 2057, 2097, 2393, 9611, 2115], [7592, 0, 0, 0, 0, 0]])\n>>> output = model(input_ids)\n>>> print(output.logits)\ntensor([[ 0.0082, -0.2307],\n        [ 0.1317, -0.1683]], grad_fn=<AddmmBackward0>)\n```\n\nHere is the actual output of the second sequence:\n\n```py\n>>> input_ids = torch.tensor([[7592]])\n>>> output = model(input_ids)\n>>> print(output.logits)\ntensor([[-0.1008, -0.4061]], grad_fn=<AddmmBackward0>)\n```\n\nMost of the time, you should provide an `attention_mask` to your model to ignore the padding tokens to avoid this silent error. Now the output of the second sequence matches its actual output:\n\n<Tip>\n\nBy default, the tokenizer creates an `attention_mask` for you based on your specific tokenizer's defaults.\n\n</Tip>\n\n```py\n>>> attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1], [1, 0, 0, 0, 0, 0]])\n>>> output = model(input_ids, attention_mask=attention_mask)\n>>> print(output.logits)\ntensor([[ 0.0082, -0.2307],\n        [-0.1008, -0.4061]], grad_fn=<AddmmBackward0>)\n```\n\n\ud83e\udd17 Transformers doesn't automatically create an `attention_mask` to mask a padding token if it is provided because:\n\n- Some models don't have a padding token.\n- For some use-cases, users want a model to attend to a padding token.\n\n## ValueError: Unrecognized configuration class XYZ for this kind of AutoModel\n\nGenerally, we recommend using the [`AutoModel`] class to load pretrained instances of models. This class\ncan automatically infer and load the correct architecture from a given checkpoint based on the configuration. If you see\nthis `ValueError` when loading a model from a checkpoint, this means the Auto class couldn't find a mapping from\nthe configuration in the given checkpoint to the kind of model you are trying to load. Most commonly, this happens when a\ncheckpoint doesn't support a given task."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 9c91336f-b6ba-4a00-b1ca-3a9c9ffc7c6b
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/troubleshooting", "title": "Troubleshoot", "tokens": 2128, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"9c91336f-b6ba-4a00-b1ca-3a9c9ffc7c6b\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f574da76-3507-4737-9ba1-f03f993b1b06\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"cf7ceb63fcd69053af31c31e769d7cbe115106f05438ffa92b3abb285b66d3aa\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6854394c-ac5f-4f21-940d-feca2c3310ac\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/troubleshooting\", \"title\": \"Troubleshoot\", \"tokens\": 2128, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"597354f279b4e601ba3008b16676ceaa7fb27c8b3aaaac36042bd6b73758807b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7139, \"end_char_idx\": 9048, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "ref_doc_id": "f574da76-3507-4737-9ba1-f03f993b1b06", "chroma:document": "Now the output of the second sequence matches its actual output:\n\n<Tip>\n\nBy default, the tokenizer creates an `attention_mask` for you based on your specific tokenizer's defaults.\n\n</Tip>\n\n```py\n>>> attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1], [1, 0, 0, 0, 0, 0]])\n>>> output = model(input_ids, attention_mask=attention_mask)\n>>> print(output.logits)\ntensor([[ 0.0082, -0.2307],\n        [-0.1008, -0.4061]], grad_fn=<AddmmBackward0>)\n```\n\n\ud83e\udd17 Transformers doesn't automatically create an `attention_mask` to mask a padding token if it is provided because:\n\n- Some models don't have a padding token.\n- For some use-cases, users want a model to attend to a padding token.\n\n## ValueError: Unrecognized configuration class XYZ for this kind of AutoModel\n\nGenerally, we recommend using the [`AutoModel`] class to load pretrained instances of models. This class\ncan automatically infer and load the correct architecture from a given checkpoint based on the configuration. If you see\nthis `ValueError` when loading a model from a checkpoint, this means the Auto class couldn't find a mapping from\nthe configuration in the given checkpoint to the kind of model you are trying to load. Most commonly, this happens when a\ncheckpoint doesn't support a given task.\nFor instance, you'll see this error in the following example because there is no GPT2 for question answering:\n\n```py\n>>> from transformers import AutoProcessor, AutoModelForQuestionAnswering\n\n>>> processor = AutoProcessor.from_pretrained(\"openai-community/gpt2-medium\")\n>>> model = AutoModelForQuestionAnswering.from_pretrained(\"openai-community/gpt2-medium\")\nValueError: Unrecognized configuration class <class 'transformers.models.gpt2.configuration_gpt2.GPT2Config'> for this kind of AutoModel: AutoModelForQuestionAnswering.\nModel type should be one of AlbertConfig, BartConfig, BertConfig, BigBirdConfig, BigBirdPegasusConfig, BloomConfig, ...\n```"}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 1a309e02-c931-4334-9524-4f525ba05aab
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/serialization", "title": "Export to ONNX", "tokens": 2388, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"1a309e02-c931-4334-9524-4f525ba05aab\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fa46a645-8e6b-465b-ae69-3d51f650c846\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b97638da20ee8948778fc11261763dd7a2666a79a5866a41f99ad4299f694ae6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5836f719-6e02-4e19-8893-47bbef5b5e90\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"931116e3deacf06a9e2f17d13ea7291f63b37d84bf62f1bea4a600e10f1d1523\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2694, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "ref_doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "chroma:document": "# Export to ONNX\n\nDeploying \ud83e\udd17 Transformers models in production environments often requires, or can benefit from exporting the models into \na serialized format that can be loaded and executed on specialized runtimes and hardware.\n\n\ud83e\udd17 Optimum is an extension of Transformers that enables exporting models from PyTorch or TensorFlow to serialized formats \nsuch as ONNX and TFLite through its `exporters` module. \ud83e\udd17 Optimum also provides a set of performance optimization tools to train \nand run models on targeted hardware with maximum efficiency.\n\nThis guide demonstrates how you can export \ud83e\udd17 Transformers models to ONNX with \ud83e\udd17 Optimum, for the guide on exporting models to TFLite, \nplease refer to the [Export to TFLite page](tflite).\n\n## Export to ONNX \n\n[ONNX (Open Neural Network eXchange)](http://onnx.ai) is an open standard that defines a common set of operators and a \ncommon file format to represent deep learning models in a wide variety of frameworks, including PyTorch and\nTensorFlow. When a model is exported to the ONNX format, these operators are used to\nconstruct a computational graph (often called an _intermediate representation_) which\nrepresents the flow of data through the neural network.\n\nBy exposing a graph with standardized operators and data types, ONNX makes it easy to\nswitch between frameworks. For example, a model trained in PyTorch can be exported to\nONNX format and then imported in TensorFlow (and vice versa).\n\nOnce exported to ONNX format, a model can be:\n- optimized for inference via techniques such as [graph optimization](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization) and [quantization](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/quantization). \n- run with ONNX Runtime via [`ORTModelForXXX` classes](https://huggingface.co/docs/optimum/onnxruntime/package_reference/modeling_ort),\nwhich follow the same `AutoModel` API as the one you are used to in \ud83e\udd17 Transformers.\n- run with [optimized inference pipelines](https://huggingface.co/docs/optimum/main/en/onnxruntime/usage_guides/pipelines),\nwhich has the same API as the [`pipeline`] function in \ud83e\udd17 Transformers. \n\n\ud83e\udd17 Optimum provides support for the ONNX export by leveraging configuration objects. These configuration objects come \nready-made for a number of model architectures, and are designed to be easily extendable to other architectures.\n\nFor the list of ready-made configurations, please refer to [\ud83e\udd17 Optimum documentation](https://huggingface.co/docs/optimum/exporters/onnx/overview).\n\nThere are two ways to export a \ud83e\udd17 Transformers model to ONNX, here we show both:\n\n- export with \ud83e\udd17 Optimum via CLI.\n- export with \ud83e\udd17 Optimum with `optimum.onnxruntime`."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 5836f719-6e02-4e19-8893-47bbef5b5e90
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/serialization", "title": "Export to ONNX", "tokens": 2388, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"5836f719-6e02-4e19-8893-47bbef5b5e90\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fa46a645-8e6b-465b-ae69-3d51f650c846\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b97638da20ee8948778fc11261763dd7a2666a79a5866a41f99ad4299f694ae6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1a309e02-c931-4334-9524-4f525ba05aab\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"4a53a6ddf954af28b05288e06c33c5f1c0f3437a8886a02e96d15c3f5fd3ba26\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"80d8b397-2a54-4673-a046-c84516e2c1c3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"345ffc7a2848e26b8c9f520076bb4fc57d3fd19f076f6ca2a0b39a6903913f41\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 994, \"end_char_idx\": 4083, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "ref_doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "chroma:document": "When a model is exported to the ONNX format, these operators are used to\nconstruct a computational graph (often called an _intermediate representation_) which\nrepresents the flow of data through the neural network.\n\nBy exposing a graph with standardized operators and data types, ONNX makes it easy to\nswitch between frameworks. For example, a model trained in PyTorch can be exported to\nONNX format and then imported in TensorFlow (and vice versa).\n\nOnce exported to ONNX format, a model can be:\n- optimized for inference via techniques such as [graph optimization](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/optimization) and [quantization](https://huggingface.co/docs/optimum/onnxruntime/usage_guides/quantization). \n- run with ONNX Runtime via [`ORTModelForXXX` classes](https://huggingface.co/docs/optimum/onnxruntime/package_reference/modeling_ort),\nwhich follow the same `AutoModel` API as the one you are used to in \ud83e\udd17 Transformers.\n- run with [optimized inference pipelines](https://huggingface.co/docs/optimum/main/en/onnxruntime/usage_guides/pipelines),\nwhich has the same API as the [`pipeline`] function in \ud83e\udd17 Transformers. \n\n\ud83e\udd17 Optimum provides support for the ONNX export by leveraging configuration objects. These configuration objects come \nready-made for a number of model architectures, and are designed to be easily extendable to other architectures.\n\nFor the list of ready-made configurations, please refer to [\ud83e\udd17 Optimum documentation](https://huggingface.co/docs/optimum/exporters/onnx/overview).\n\nThere are two ways to export a \ud83e\udd17 Transformers model to ONNX, here we show both:\n\n- export with \ud83e\udd17 Optimum via CLI.\n- export with \ud83e\udd17 Optimum with `optimum.onnxruntime`.\n\n### Exporting a \ud83e\udd17 Transformers model to ONNX with CLI\n\nTo export a \ud83e\udd17 Transformers model to ONNX, first install an extra dependency:\n\n```bash\npip install optimum[exporters]\n```\n\nTo check out all available arguments, refer to the [\ud83e\udd17 Optimum docs](https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model#exporting-a-model-to-onnx-using-the-cli), \nor view help in command line:\n\n```bash\noptimum-cli export onnx --help\n```\n\nTo export a model's checkpoint from the \ud83e\udd17 Hub, for example, `distilbert/distilbert-base-uncased-distilled-squad`, run the following command: \n\n```bash\noptimum-cli export onnx --model distilbert/distilbert-base-uncased-distilled-squad distilbert_base_uncased_squad_onnx/\n```\n\nYou should see the logs indicating progress and showing where the resulting `model.onnx` is saved, like this:\n\n```bash\nValidating ONNX model distilbert_base_uncased_squad_onnx/model.onnx...\n\t-[\u2713] ONNX model output names match reference model (start_logits, end_logits)\n\t- Validating ONNX Model output \"start_logits\":\n\t\t-[\u2713] (2, 16) matches (2, 16)\n\t\t-[\u2713] all values close (atol: 0.0001)\n\t- Validating ONNX Model output \"end_logits\":\n\t\t-[\u2713] (2, 16) matches (2, 16)\n\t\t-[\u2713] all values close (atol: 0.0001)\nThe ONNX export succeeded and the exported model was saved at: distilbert_base_uncased_squad_onnx\n```\n\nThe example above illustrates exporting a checkpoint from \ud83e\udd17 Hub."}


created_at: 2024-07-28 18:57:50
topic: persistent://default/default/5523a7e1-1ad4-449b-b059-70d441f3d0d1
id: 80d8b397-2a54-4673-a046-c84516e2c1c3
encoding: FLOAT32
metadata:
{"url": "https://huggingface.co/docs/transformers/serialization", "title": "Export to ONNX", "tokens": 2388, "retrieve_doc": true, "source": "transformers", "_node_content": "{\"id_\": \"80d8b397-2a54-4673-a046-c84516e2c1c3\", \"embedding\": null, \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"fa46a645-8e6b-465b-ae69-3d51f650c846\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"b97638da20ee8948778fc11261763dd7a2666a79a5866a41f99ad4299f694ae6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5836f719-6e02-4e19-8893-47bbef5b5e90\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://huggingface.co/docs/transformers/serialization\", \"title\": \"Export to ONNX\", \"tokens\": 2388, \"retrieve_doc\": true, \"source\": \"transformers\"}, \"hash\": \"1e1f4e9fa7bb3fd1839a3571e8d4ab7bf9438c1d296500a931de4c48a3c82dbf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3280cd42-0b2b-4c88-aeee-0385d7c0d3f3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7d11f21fa86f5366da58a521326e1d04781c99acdd252ed4c285ae7f564a002a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2696, \"end_char_idx\": 5498, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "ref_doc_id": "fa46a645-8e6b-465b-ae69-3d51f650c846", "chroma:document": "### Exporting a \ud83e\udd17 Transformers model to ONNX with CLI\n\nTo export a \ud83e\udd17 Transformers model to ONNX, first install an extra dependency:\n\n```bash\npip install optimum[exporters]\n```\n\nTo check out all available arguments, refer to the [\ud83e\udd17 Optimum docs](https://huggingface.co/docs/optimum/exporters/onnx/usage_guides/export_a_model#exporting-a-model-to-onnx-using-the-cli), \nor view help in command line:\n\n```bash\noptimum-cli export onnx --help\n```\n\nTo export a model's checkpoint from the \ud83e\udd17 Hub, for example, `distilbert/distilbert-base-uncased-distilled-squad`, run the following command: \n\n```bash\noptimum-cli export onnx --model distilbert/distilbert-base-uncased-distilled-squad distilbert_base_uncased_squad_onnx/\n```\n\nYou should see the logs indicating progress and showing where the resulting `model.onnx` is saved, like this:\n\n```bash\nValidating ONNX model distilbert_base_uncased_squad_onnx/model.onnx...\n\t-[\u2713] ONNX model output names match reference model (start_logits, end_logits)\n\t- Validating ONNX Model output \"start_logits\":\n\t\t-[\u2713] (2, 16) matches (2, 16)\n\t\t-[\u2713] all values close (atol: 0.0001)\n\t- Validating ONNX Model output \"end_logits\":\n\t\t-[\u2713] (2, 16) matches (2, 16)\n\t\t-[\u2713] all values close (atol: 0.0001)\nThe ONNX export succeeded and the exported model was saved at: distilbert_base_uncased_squad_onnx\n```\n\nThe example above illustrates exporting a checkpoint from \ud83e\udd17 Hub. When exporting a local model, first make sure that you \nsaved both the model's weights and tokenizer files in the same directory (`local_path`). When using CLI, pass the \n`local_path` to the `model` argument instead of the checkpoint name on \ud83e\udd17 Hub and provide the `--task` argument. \nYou can review the list of supported tasks in the [\ud83e\udd17 Optimum documentation](https://huggingface.co/docs/optimum/exporters/task_manager).\nIf `task` argument is not provided, it will default to the model architecture without any task specific head.\n\n```bash\noptimum-cli export onnx --model local_path --task question-answering distilbert_base_uncased_squad_onnx/\n```\n\nThe resulting `model.onnx` file can then be run on one of the [many\naccelerators](https://onnx.ai/supported-tools.html#deployModel) that support the ONNX\nstandard. For example, we can load and run the model with [ONNX\nRuntime](https://onnxruntime.ai/) as follows:\n\n```python\n>>> from transformers import AutoTokenizer\n>>> from optimum.onnxruntime import ORTModelForQuestionAnswering\n\n>>> tokenizer = AutoTokenizer.from_pretrained(\"distilbert_base_uncased_squad_onnx\")\n>>> model = ORTModelForQuestionAnswering.from_pretrained(\"distilbert_base_uncased_squad_onnx\")\n>>> inputs = tokenizer(\"What am I using?\", \"Using DistilBERT with ONNX Runtime!\", return_tensors=\"pt\")\n>>> outputs = model(**inputs)\n```\n\nThe process is identical for TensorFlow checkpoints on the Hub."}


