# embedding_metadata

key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "c321ddbc-bfd5-468c-9c80-b09e29b4d95c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4390dab4-fc7f-4308-88d8-3bc5236a8ddc", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "7028f0a71eb4eaa078875dc20081b85c847f3a8b886d3bf8fd45b5fbf0315dd2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d29664b4-a699-4d56-8f9e-57062feecd6d", "node_type": "1", "metadata": {}, "hash": "ac94ea20c8bd6a807f693f1093b7e5e8417e961abb38b02d75b5d6dfee70bfe7", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 130155, "end_char_idx": 130928, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "d29664b4-a699-4d56-8f9e-57062feecd6d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c321ddbc-bfd5-468c-9c80-b09e29b4d95c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "214cdf2c3d33c836e3b49ee5ecf8bf7fb6bd49b8e7bd83dca98bd5ea61db27b6", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "388d94ab-cb0f-49f5-8fa9-3652d2dcd0c4", "node_type": "1", "metadata": {}, "hash": "de6cce4e7a26ce7e60d5426e071bd754c34d851ec8fb69098a2446129e3cd224", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 130528, "end_char_idx": 131301, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "388d94ab-cb0f-49f5-8fa9-3652d2dcd0c4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d29664b4-a699-4d56-8f9e-57062feecd6d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "09721214d98190fa42bb05388243fbd082a635759db21430d48506d78243b6b1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "281bf965-9735-48f1-9060-d4917ef69541", "node_type": "1", "metadata": {}, "hash": "477576c707c14dd212635e23f09461d14658dc4a40211c01c6cdcc7957e8a7bc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 130901, "end_char_idx": 131674, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "281bf965-9735-48f1-9060-d4917ef69541", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "388d94ab-cb0f-49f5-8fa9-3652d2dcd0c4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "733d9b5ec8dd49ee0901a4602878335a9b8259eeb48f4b7953b27873c27ceefb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "06165489-bcfe-41ce-aae7-3d25422ddc45", "node_type": "1", "metadata": {}, "hash": "2fc47a33a114b235068e2f17791c1b30b08f6fc620e98c06027d17d2404ee7ec", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 131274, "end_char_idx": 132047, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "06165489-bcfe-41ce-aae7-3d25422ddc45", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "281bf965-9735-48f1-9060-d4917ef69541", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "02e77b6f9be005e384924015938e1755019924220493335545f68ace8fc86a7d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b57f6747-6689-45a2-b770-15aac2d9171a", "node_type": "1", "metadata": {}, "hash": "f10b8c7c0df590b0a3f25e88342f49f7b0186810efb6628dccaab6cc7ac65292", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 131647, "end_char_idx": 132420, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "b57f6747-6689-45a2-b770-15aac2d9171a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "06165489-bcfe-41ce-aae7-3d25422ddc45", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "60da3efc7691266b7ceac1b39dbe54fae2743fe462d2c01f7e9a36f8d21fda34", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4696b62b-7b37-4d50-944a-708d2fa7293d", "node_type": "1", "metadata": {}, "hash": "ef44c0afb7b121f5ec796c5969819c4565753f35908f06abd0f10b6e03929af6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 132020, "end_char_idx": 132793, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "4696b62b-7b37-4d50-944a-708d2fa7293d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b57f6747-6689-45a2-b770-15aac2d9171a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "5e46fa889e437d98c357e62ceabc8d7e99a4a2cf4ef54e7c29bee37687321540", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fb4a2c07-e136-4b9e-aaa5-de9ec6efa1e6", "node_type": "1", "metadata": {}, "hash": "0557c06fb9bb33f261598a2a82c8e924ead159f6a7aeec65eec64b11993c086c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 132393, "end_char_idx": 133166, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "fb4a2c07-e136-4b9e-aaa5-de9ec6efa1e6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4696b62b-7b37-4d50-944a-708d2fa7293d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "cf60fcc3e4d2acf8601b748fc74a26d57ec69cbc4c26fcd6ff80de2b2234bb55", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "76dd6a9b-7df8-45b5-b8e0-0b7d4a2482e5", "node_type": "1", "metadata": {}, "hash": "57c85a53c019299817ecaeda659078dd4fc237fa180c8449d02cda41bbbc09a9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 132766, "end_char_idx": 133539, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "76dd6a9b-7df8-45b5-b8e0-0b7d4a2482e5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fb4a2c07-e136-4b9e-aaa5-de9ec6efa1e6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "30a4ccbcc9f5a63d48f2bf9dfe28ded5f71b643b1add49a3b9a4e2621f6b9133", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a83e965c-59e6-4a34-94ee-ed996c521b3e", "node_type": "1", "metadata": {}, "hash": "8a6d302cc96e44de8254950363fb2b9a822492d591b3723579b17fb01d993594", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 133139, "end_char_idx": 133912, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "a83e965c-59e6-4a34-94ee-ed996c521b3e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "76dd6a9b-7df8-45b5-b8e0-0b7d4a2482e5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "75a74c0ee508ae773c8ff7433797fd4aef03e3e82f6d3b39dfb1c76276a88299", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "564c9966-5e25-4670-bad0-914ea62d0550", "node_type": "1", "metadata": {}, "hash": "8ca388e95c4b790a7dbad97dd14179693333730051028813589f564c49cef2dd", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 133512, "end_char_idx": 134285, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "564c9966-5e25-4670-bad0-914ea62d0550", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a83e965c-59e6-4a34-94ee-ed996c521b3e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "81fec02688760b0f027431467e491e6d87e883a90c72becd765411156c4eabe8", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e5c180c5-62ca-4850-91a6-6bca4b507cc1", "node_type": "1", "metadata": {}, "hash": "e07684f8b7f21a3244139c953b263de5e6caa6f1c5bd03410fad16da134a6457", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 133885, "end_char_idx": 134658, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "e5c180c5-62ca-4850-91a6-6bca4b507cc1", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "564c9966-5e25-4670-bad0-914ea62d0550", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b1de2d33be0ae212695054f8316896a7a536ef18715589d2e4d551b75f2046ad", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "605868a7-161a-4252-9305-6f29351d5100", "node_type": "1", "metadata": {}, "hash": "876aae8ddb133e21c8ba7cdeaf19cb15636d81183189552333d01226008a3f4b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 134258, "end_char_idx": 135031, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "605868a7-161a-4252-9305-6f29351d5100", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e5c180c5-62ca-4850-91a6-6bca4b507cc1", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3414d7b30907ef2d32fb225c2df220a3648b2f28d870e5b4a80471987a847640", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6ee0cc5f-cd66-45ae-a4a7-2f15ecbfb970", "node_type": "1", "metadata": {}, "hash": "30018fb486e861a69d74ef122240006cb1963a5d06127615534c6a01ea816aea", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 134631, "end_char_idx": 135404, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "6ee0cc5f-cd66-45ae-a4a7-2f15ecbfb970", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "605868a7-161a-4252-9305-6f29351d5100", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c6f8cf4ba65b914c1242a07be78c04c0c5f3fa2ce5aa9f81bf35bba5730b33aa", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d7bde4c8-8957-4be6-9e2c-6a50d73e38e5", "node_type": "1", "metadata": {}, "hash": "49414e5554b0c47f7a7637ffc2fa804edc460048a2a8ad8a814fa15b81b5131e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 135004, "end_char_idx": 135777, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "d7bde4c8-8957-4be6-9e2c-6a50d73e38e5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6ee0cc5f-cd66-45ae-a4a7-2f15ecbfb970", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8e925b2296d22af4a1cb1cc95b4321c21c44b62c2eef4c0cc2d365812de5643c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "74cc6790-e334-411b-8cd6-36a36a73a2ce", "node_type": "1", "metadata": {}, "hash": "0bc258b458e8e3d5336c67609a5a9d9a919c474952251381d86762fdcf449007", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 135377, "end_char_idx": 136150, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "74cc6790-e334-411b-8cd6-36a36a73a2ce", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d7bde4c8-8957-4be6-9e2c-6a50d73e38e5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2cbb98d62fa7d7dab7757fca58d6d9ac7634bd7ffbc2812cac940fdc1f03e8f0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ab8cb993-03d6-4c4a-ac2c-33735d4a5acf", "node_type": "1", "metadata": {}, "hash": "0d5a49f8353b2eb625c3506df41265bbb8909384aabd1e7407cc940cf0f982aa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 135750, "end_char_idx": 136523, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "ab8cb993-03d6-4c4a-ac2c-33735d4a5acf", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "74cc6790-e334-411b-8cd6-36a36a73a2ce", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "1f2179e34e4dd67284c160ac1b25086efb598010192274611dd570b630843880", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "79432c8e-9307-4c79-a36b-8b09688e01de", "node_type": "1", "metadata": {}, "hash": "f38a35ef644a1f9c5c8e240ab6315d8d0d419ff223b970a17c0204329b8c48fc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 136123, "end_char_idx": 136896, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "79432c8e-9307-4c79-a36b-8b09688e01de", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ab8cb993-03d6-4c4a-ac2c-33735d4a5acf", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "45bd5a7b309c4bc9e9bc0c04cca138406a61476b63e400debd9ed3bca0a32787", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ecf4f2c3-94ac-42f7-841a-125808e34da5", "node_type": "1", "metadata": {}, "hash": "7f2bc30ea11cc2ef5bce9724ef6ee09e6581e18499ef9ad9a3965ec3674f5b6b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 136496, "end_char_idx": 137269, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "ecf4f2c3-94ac-42f7-841a-125808e34da5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "79432c8e-9307-4c79-a36b-8b09688e01de", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "dc19fe38a4118816c72c6baebb2e4754e08670a824ba1d6b76187110f3ec662e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "629636df-69a5-44e7-8c7c-df247635d7f7", "node_type": "1", "metadata": {}, "hash": "fb7719c299dfd3b470d7c8d27a536e76c6cf991084920073c4eb2edcd2c1c7b9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 136869, "end_char_idx": 137642, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "629636df-69a5-44e7-8c7c-df247635d7f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ecf4f2c3-94ac-42f7-841a-125808e34da5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0672427a76ebd8d5c82cdbd7ae408432007f34f674b4f5435b69fcc7992cc9bb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "017cdd51-0d49-4aee-97d3-a6359b8f51c8", "node_type": "1", "metadata": {}, "hash": "4aacc32459bfb074f754a586146dd75164724f9c662b258c21a6376509e35a48", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 137242, "end_char_idx": 138015, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "017cdd51-0d49-4aee-97d3-a6359b8f51c8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "629636df-69a5-44e7-8c7c-df247635d7f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3f3c53c551eee28857639f0cc3554f5aed5ea10e6ba92aab5329060fb913d9a4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "48267f37-7835-4dcc-ace6-84b0050a8f9c", "node_type": "1", "metadata": {}, "hash": "5360c8f82c0de42f22659e23bcd516ecf23a35c8fa1f12687a081a80b08445db", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 137615, "end_char_idx": 138388, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "48267f37-7835-4dcc-ace6-84b0050a8f9c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "017cdd51-0d49-4aee-97d3-a6359b8f51c8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "6ad42006dfb006cefbd3be03b73e9bc1328a92dc6c253031012233c1fab60347", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c3b11e65-b7ed-48aa-b308-7e38194e2c0c", "node_type": "1", "metadata": {}, "hash": "f34ae4d66c12b89b7b5abd34b85826e176bc38e7c6c4ac56f77941ed7b7cbce1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 137988, "end_char_idx": 138761, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "c3b11e65-b7ed-48aa-b308-7e38194e2c0c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "48267f37-7835-4dcc-ace6-84b0050a8f9c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "45299a32cddafe6764f6c2916cb82ba15c79b014b2d95ee5ba7355b297331708", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ea6cae6e-95db-4369-8b7f-2da429608528", "node_type": "1", "metadata": {}, "hash": "302fe4339f9d3d73bd44372c48d509174f1b20e83f0883179cacf21038748dd3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 138361, "end_char_idx": 139134, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "ea6cae6e-95db-4369-8b7f-2da429608528", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c3b11e65-b7ed-48aa-b308-7e38194e2c0c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f2c497e4812fa681330553df83a463777ca15fa40473032f42b82019312ca6a2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3274c412-dd90-4ecc-a83e-32f6ad46a3d9", "node_type": "1", "metadata": {}, "hash": "be2c091f1fce6508f8744e896c05b2273ef38f1bfcdaae765c389d008f81a156", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 138734, "end_char_idx": 139507, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "3274c412-dd90-4ecc-a83e-32f6ad46a3d9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ea6cae6e-95db-4369-8b7f-2da429608528", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "25adaa867e1bf44102b48a90069181f76f2cb9d17b5f656f2b7f92ab59fe0e3d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "78ff4e31-84ae-4920-8165-c0b95ee577b9", "node_type": "1", "metadata": {}, "hash": "af518e055f6b8f6571a228a6b93077928c37b67bb09ae10b9504f49226023d7a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 139107, "end_char_idx": 139880, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "78ff4e31-84ae-4920-8165-c0b95ee577b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3274c412-dd90-4ecc-a83e-32f6ad46a3d9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "305853325a98f3961853e28dcc0f921c647f27490e7eb39469f44a8dc3dc32f8", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dc7c3b96-6a18-4d76-bde7-15e4858d1453", "node_type": "1", "metadata": {}, "hash": "0b8bf3184b0919f0e187bc8fe3b41056fa04bad04989004f21734b45843a4b1e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 139480, "end_char_idx": 140253, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "dc7c3b96-6a18-4d76-bde7-15e4858d1453", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "78ff4e31-84ae-4920-8165-c0b95ee577b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9319c74441f1bba3312f8e0a1a40a6ca7bd7aa33378cf1b54ad61738df2404d7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0940df6d-b435-44c6-a228-6b82d6e827af", "node_type": "1", "metadata": {}, "hash": "6fbc9ff2692d72435e8d6fa294e3f6c56820e6dfe8e7a89e3bc8d16e7188fce9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 139853, "end_char_idx": 140626, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "0940df6d-b435-44c6-a228-6b82d6e827af", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dc7c3b96-6a18-4d76-bde7-15e4858d1453", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2003669c10630d7c0299f5d702b1f6a031d18adf7ac1cf45a2a4b54aca259449", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0686cf03-b08c-42cf-8aaf-86ba0f4a9afe", "node_type": "1", "metadata": {}, "hash": "04dff4c70e97e53a8fe7642c07b3e4f8c62d97d92ce79a4cb1bc415636a98e00", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 140226, "end_char_idx": 140999, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "0686cf03-b08c-42cf-8aaf-86ba0f4a9afe", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0940df6d-b435-44c6-a228-6b82d6e827af", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "d8c40a576bb3beb7e9cfe0be19520292b888a66f31aa76dcf7b56d720fbe0e48", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f0775ab8-484f-45c9-9133-f93be1eca451", "node_type": "1", "metadata": {}, "hash": "60fcdc3a95953f6ab32a381b91a9588efeb8bd35b4f53599c4df1b8d7915b942", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 140599, "end_char_idx": 141372, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "f0775ab8-484f-45c9-9133-f93be1eca451", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0686cf03-b08c-42cf-8aaf-86ba0f4a9afe", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8911f6fd3c96d69d534fce5630fc0f79f4a8f06b79b4afa81662c4533b73ab26", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d46cbe5a-b56e-428e-9994-8de4167a7d38", "node_type": "1", "metadata": {}, "hash": "721180a06a20cf404a327e1a7d19c0aeb079e3b95fc3b8b3e322854dfe4c4e12", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 140972, "end_char_idx": 141745, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "d46cbe5a-b56e-428e-9994-8de4167a7d38", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f0775ab8-484f-45c9-9133-f93be1eca451", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c069989429615819f100b958f2aa084d355f2846eab5b3e1a8316ddbde412ba9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f43cd5f5-b6de-4aff-badd-e895e2ee69cb", "node_type": "1", "metadata": {}, "hash": "0bfd9d39b2b3f52fcd697e781c467ff0330bb4bac9e3834eb5dd53c55730d0bb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 141345, "end_char_idx": 142118, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "f43cd5f5-b6de-4aff-badd-e895e2ee69cb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d46cbe5a-b56e-428e-9994-8de4167a7d38", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c0eef2be7f14e8b7c853ed81e133346b8d13f9076c0f1fc54c8b854df37c6962", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dbff6b83-5602-40e5-a0da-b6a0985ec0c7", "node_type": "1", "metadata": {}, "hash": "83d4fbc9ab2f1df97778e83fc76c4e709dbc676f2c5be5ccc4bf795c9f646b15", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 141718, "end_char_idx": 142491, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "dbff6b83-5602-40e5-a0da-b6a0985ec0c7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f43cd5f5-b6de-4aff-badd-e895e2ee69cb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9a7f9d0069e91708c161aa7b2d2862fbf39f34c1fb06ee6c760dac49c6fdc6cd", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5e92443f-649a-4037-91ed-152d94c8a4e5", "node_type": "1", "metadata": {}, "hash": "cdafcaf07370f42b88620dd5755727ca4e4e4a17a5976406cfd8c2dc927cc289", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 142091, "end_char_idx": 142864, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "5e92443f-649a-4037-91ed-152d94c8a4e5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dbff6b83-5602-40e5-a0da-b6a0985ec0c7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f3644aef3bd83efa222c0f1310a1fd6625e9af89b417a605f7500186c9376a47", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a07bfd56-1c05-4531-b1e1-c1bcb0c1f813", "node_type": "1", "metadata": {}, "hash": "66b3ba24dfda6dcc2125fee7b8bc7a58d9f3016a3c41dba48afbe63f898086d1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 142464, "end_char_idx": 143237, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "a07bfd56-1c05-4531-b1e1-c1bcb0c1f813", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5e92443f-649a-4037-91ed-152d94c8a4e5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a2386ea6124e9607c40189b236c4dc328507a4072f9c7e8c253318f8bfbf6631", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "65f4036c-e67f-42a6-8789-73f35fefd2a8", "node_type": "1", "metadata": {}, "hash": "c0e92cd714345833aa97e00e516bdf56df1e426da560d2e5c11b85fb79a6faf5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 142837, "end_char_idx": 143610, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "65f4036c-e67f-42a6-8789-73f35fefd2a8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a07bfd56-1c05-4531-b1e1-c1bcb0c1f813", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "88aca8383d2dfbfaeab589509b70a571d4cb8f98258d084d88d6670661dd4f27", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "aa587342-3767-44bb-9349-4e39365248ff", "node_type": "1", "metadata": {}, "hash": "d5f06b2b4b373add8f71e47dbbd78eb4953890870e49a43c5449d3fcaba8d990", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 143210, "end_char_idx": 143983, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "aa587342-3767-44bb-9349-4e39365248ff", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "65f4036c-e67f-42a6-8789-73f35fefd2a8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0307b15e944acdc8e1a5b52b9c19726cb208e4a2e9cf08147bc3cce44a324c80", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a79eb045-70bb-4f56-a568-26b526478684", "node_type": "1", "metadata": {}, "hash": "432191d8b9a5ad7d61238c20969b638fbeeae3c5d91c266252c348cfc6e80c8a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 143583, "end_char_idx": 144356, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "a79eb045-70bb-4f56-a568-26b526478684", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "aa587342-3767-44bb-9349-4e39365248ff", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c9a29869843789670926f7a437dfd240e0a6e19af00568240a023700fe5d13d5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "860ca1e0-eece-41e2-b081-8a4f90c519b9", "node_type": "1", "metadata": {}, "hash": "8888534ed31607300b57770dba8a20c66138b89dced4e7037a25ae9536eb8501", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 143956, "end_char_idx": 144729, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "860ca1e0-eece-41e2-b081-8a4f90c519b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a79eb045-70bb-4f56-a568-26b526478684", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "65af3bca7c1ae1b6b4fc22c791adb03d1fa7b020d3cb49e22abbd7866b3e36bf", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6a3d1ead-82ed-4520-b15f-2f9ea3338dec", "node_type": "1", "metadata": {}, "hash": "bd6a28ca0d40b14b7bafafd66b52a8baaa3b7a458985730179786731ac44ac21", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 144329, "end_char_idx": 145102, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "6a3d1ead-82ed-4520-b15f-2f9ea3338dec", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "860ca1e0-eece-41e2-b081-8a4f90c519b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6f20669d748256440d058e10a8b069ee022458ec66a7f58e1e74f14d2d06bc3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6b0b28ae-d33c-4764-882e-05d4ea5051ea", "node_type": "1", "metadata": {}, "hash": "fbf340ea2cd006cd3fbc9d2522878ded528947ad9be6ac3ba28f7b55fb965390", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 144702, "end_char_idx": 145475, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "6b0b28ae-d33c-4764-882e-05d4ea5051ea", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6a3d1ead-82ed-4520-b15f-2f9ea3338dec", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "5d41c6b379ac846ef1055e9c6dea99879f4391786da02d5074930adb82cb0301", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d61b54f1-1641-4b8a-a8e1-d36cb663f569", "node_type": "1", "metadata": {}, "hash": "b16759632ced258ad0552f754e0aed59f85e2af63c8b5795b63c8387e770fc06", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 145075, "end_char_idx": 145848, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "d61b54f1-1641-4b8a-a8e1-d36cb663f569", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6b0b28ae-d33c-4764-882e-05d4ea5051ea", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "e64b3b0462b1d646784f9f97c5e8cc0f665c0bddba503768ed7ccb9704e56100", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3ab1a2dc-e635-4d7e-9b4b-a88b487f0d6c", "node_type": "1", "metadata": {}, "hash": "c5f7c80e9ab92cc723bae96177889e7e0ea1c3ef9727fb0e32540cef8bcaa583", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 145448, "end_char_idx": 146221, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "3ab1a2dc-e635-4d7e-9b4b-a88b487f0d6c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d61b54f1-1641-4b8a-a8e1-d36cb663f569", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "1b791d69ffd00628bae23a8b2aa0d58117f2325b96ed4e914e26d747e9f2a7c8", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "14a7a27a-105a-48ab-906b-6a5d671644cf", "node_type": "1", "metadata": {}, "hash": "b5ab862c6a2492d48ab4866970ed2496a0239fe501e4e95d21654456bfbb7045", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 145821, "end_char_idx": 146594, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "14a7a27a-105a-48ab-906b-6a5d671644cf", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3ab1a2dc-e635-4d7e-9b4b-a88b487f0d6c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2b20370d548e3646dd14a7b0ffeec1842e8eeebf213f06ead34464ec007e3c27", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "15c19996-390b-4ef6-9bfd-4d6bfc539779", "node_type": "1", "metadata": {}, "hash": "54c0d914d705f813fc78941262d46e94b443741ee7ed64d142640b1408d85e84", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 146194, "end_char_idx": 146967, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "15c19996-390b-4ef6-9bfd-4d6bfc539779", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "14a7a27a-105a-48ab-906b-6a5d671644cf", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6be1ec2a313a465eb23bfea16585a2695a6cc0a46ce365ed7c816a77832088e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "cbf09828-aa8a-4350-b8e2-a84c942f2643", "node_type": "1", "metadata": {}, "hash": "d5a8c487ab1f26dbf007933228c5feb573de559eaf2fe0ee28e34c6771f68569", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 146567, "end_char_idx": 147340, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "cbf09828-aa8a-4350-b8e2-a84c942f2643", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "15c19996-390b-4ef6-9bfd-4d6bfc539779", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "81679943079d84f2969291cc40aef5f69b6d522c3b8597e968d5ec16ab3edde3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bcbf8954-3214-4027-9617-a2a3db9c7c8c", "node_type": "1", "metadata": {}, "hash": "eb786dbc04a0d621cb5903290817cda9e339e36029fdb2ea776db8e66e6c9bd0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 146940, "end_char_idx": 147713, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "bcbf8954-3214-4027-9617-a2a3db9c7c8c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "cbf09828-aa8a-4350-b8e2-a84c942f2643", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "7461cb7a98636a8d6ebdc1967814c1f6b18236eee41cf0fbb9f0830f119b4642", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0a9d0e35-8a97-4254-b8a2-7e2e9a3670b5", "node_type": "1", "metadata": {}, "hash": "4ddab79f5e3912c66f7aa5c0fe78bc4351c9969f891af9344c5dc1b32c98ee77", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 147313, "end_char_idx": 148086, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "0a9d0e35-8a97-4254-b8a2-7e2e9a3670b5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "bcbf8954-3214-4027-9617-a2a3db9c7c8c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b824eecaa3f0f29981017ba1b511ce6f43cd993f1245a884ff8c09b393ad9c1f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b406d271-60b4-4314-b987-172ba816ddc4", "node_type": "1", "metadata": {}, "hash": "4cf48e2bdad0f02e18687d227890286f3a6823ef0650c3f163d3f318aa743770", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 147686, "end_char_idx": 148459, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "b406d271-60b4-4314-b987-172ba816ddc4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0a9d0e35-8a97-4254-b8a2-7e2e9a3670b5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ff9bc4444d56ea8a1d058171675feb694dd3bf62363e75294ce40e018c926364", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9c384b7c-3835-42d1-8838-f16c8a8361f9", "node_type": "1", "metadata": {}, "hash": "a2a02cb6a6d781c9040964425b55746e5b7cd973412d19065dbd4bd7ed146ec2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 148059, "end_char_idx": 148832, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "9c384b7c-3835-42d1-8838-f16c8a8361f9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b406d271-60b4-4314-b987-172ba816ddc4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "13336cde5aad4e1ca82433293f943aabcda590f80ab5283e682be5ad941c9416", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1281c51d-7ae3-4356-9ed8-bb3117764157", "node_type": "1", "metadata": {}, "hash": "678919dd41ed8d8bc5f30ea698b50a0a63ebbc44bed5c0791903199752956084", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 148432, "end_char_idx": 149205, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "1281c51d-7ae3-4356-9ed8-bb3117764157", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9c384b7c-3835-42d1-8838-f16c8a8361f9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "06257922cf34e805752bec80cedd697a170a3a2de498b6e6f0296831494ff387", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "24261ab1-4c3c-4862-8d81-aeac31d052f5", "node_type": "1", "metadata": {}, "hash": "8a20273f38e3af2162798993a7994225960d2496aea5f285198ad74cfb6f95a6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 148805, "end_char_idx": 149578, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "24261ab1-4c3c-4862-8d81-aeac31d052f5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1281c51d-7ae3-4356-9ed8-bb3117764157", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "29dd88d9c9474f1ed2943d49462bd0d602eae2992f37fc62be0000f0faa70ae5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3228b314-9abb-45c8-9a11-0de4a08dc582", "node_type": "1", "metadata": {}, "hash": "9665259e8efc87cf91086d993557b9a7718c608dc3749237665fd984d7a79fc0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 149178, "end_char_idx": 149951, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "3228b314-9abb-45c8-9a11-0de4a08dc582", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "24261ab1-4c3c-4862-8d81-aeac31d052f5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3993205c5722516790c4786ba7756b043dcf5a01db2321a4c5ba8b91a942a40f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7515ffa8-55ba-41bc-85bb-ae0a8d3ab429", "node_type": "1", "metadata": {}, "hash": "6ab7d17120b678e717010b9a37f60827f59fdf58c864fd7eccf12f5cf7cd6caf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 149551, "end_char_idx": 150328, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:


key: _node_content
string_value:
{"id_": "7515ffa8-55ba-41bc-85bb-ae0a8d3ab429", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "962d4118-13fe-4de1-a528-5a4cc59a19a0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f5f29ee5958ac7f50fe2351cdbf79f4f368ead1908c66bad45aad0c0b6fe5d3a", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3228b314-9abb-45c8-9a11-0de4a08dc582", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Visualizing_embeddings_in_3D.ipynb", "title": "Visualizing embeddings in 3D", "tokens": 104116, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "566d043487372ca3f0dd450bdbbfb1022d5f8bad0c1e787216bad0c3f21add56", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 149921, "end_char_idx": 150338, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</div>


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb


key: _node_content
string_value:
{"id_": "7e71a18e-66c6-4cf4-9abd-a23164ec84ab", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ebee59c7-fa7d-4c1c-96fb-96105c80601e", "node_type": "1", "metadata": {}, "hash": "8ffd0304bd6848014b2e1ffc037b57605779b88442030a8acda8795db963af49", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2014, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Multiclass Classification for Transactions

For this notebook we will be looking to classify a public dataset of transactions into a number of categories that we have predefined. These approaches should be replicable to any multiclass classification use case where we are trying to fit transactional data into predefined categories, and by the end of running through this you should have a few approaches for dealing with both labelled and unlabelled datasets.

The different approaches we'll be taking in this notebook are:
- **Zero-shot Classification:** First we'll do zero shot classification to put transactions in one of five named buckets using only a prompt for guidance
- **Classification with Embeddings:** Following this we'll create embeddings on a labelled dataset, and then use a traditional classification model to test their effectiveness at identifying our categories
- **Fine-tuned Classification:** Lastly we'll produce a fine-tuned model trained on our labelled dataset to see how this compares to the zero-shot and few-shot classification approaches

## Setup


```python
%load_ext autoreload
%autoreload
%pip install openai 'openai[datalib]' 'openai[embeddings]' transformers

```


```python
import openai
import pandas as pd
import numpy as np
import json
import os

COMPLETIONS_MODEL = "gpt-4"

client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if you didn't set as an env var>"))
```

### Load dataset

We're using a public transaction dataset of transactions over £25k for the Library of Scotland. The dataset has three features that we'll be using:
- Supplier: The name of the supplier
- Description: A text description of the transaction
- Value: The value of the transaction in GBP

**Source**:

https://data.nls.uk/data/organisational-data/transactions-over-25k/


```python
transactions = pd.read_csv('./data/25000_spend_dataset_current.csv', encoding= 'unicode_escape')
len(transactions)

```




    359




```python
transactions.head()

```


key: _node_content
string_value:
{"id_": "ebee59c7-fa7d-4c1c-96fb-96105c80601e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7e71a18e-66c6-4cf4-9abd-a23164ec84ab", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2b3fd303e8327cdca94e269746f79e74018a6ae5b36237a5960288abb92ba162", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0d33e960-07af-401f-ab60-1c9e37eec2d9", "node_type": "1", "metadata": {}, "hash": "d1ac6c52bd1302cc21a20975e5c9cc202da2353b152164beca81af621c767242", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1085, "end_char_idx": 3313, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
%load_ext autoreload
%autoreload
%pip install openai 'openai[datalib]' 'openai[embeddings]' transformers

```


```python
import openai
import pandas as pd
import numpy as np
import json
import os

COMPLETIONS_MODEL = "gpt-4"

client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if you didn't set as an env var>"))
```

### Load dataset

We're using a public transaction dataset of transactions over £25k for the Library of Scotland. The dataset has three features that we'll be using:
- Supplier: The name of the supplier
- Description: A text description of the transaction
- Value: The value of the transaction in GBP

**Source**:

https://data.nls.uk/data/organisational-data/transactions-over-25k/


```python
transactions = pd.read_csv('./data/25000_spend_dataset_current.csv', encoding= 'unicode_escape')
len(transactions)

```




    359




```python
transactions.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>21/04/2016</td>
      <td>M &amp; J Ballantyne Ltd</td>
      <td>George IV Bridge Work</td>
      <td>35098.0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>26/04/2016</td>
      <td>Private Sale</td>
      <td>Literary &amp; Archival Items</td>
      <td>30000.0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>30/04/2016</td>
      <td>City Of Edinburgh Council</td>
      <td>Non Domestic Rates</td>
      <td>40800.0</td>
    </tr>
    <tr>
      <th>3</th>
      <td>09/05/2016</td>
      <td>Computacenter Uk</td>
      <td>Kelvin Hall</td>
      <td>72835.0</td>
    </tr>
    <tr>
      <th>4</th>
      <td>09/05/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>64361.0</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "0d33e960-07af-401f-ab60-1c9e37eec2d9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ebee59c7-fa7d-4c1c-96fb-96105c80601e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "591fe110d816e37284cb3e8196b127eb44f30ec5002635d85cdf7a949e165139", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7652d4d8-4e52-4c40-93d4-09d2d9c2608d", "node_type": "1", "metadata": {}, "hash": "50fb1c9f736c9735ec9c8f55d6394041f028a14cbe35da956227997adb4ca7ce", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3318, "end_char_idx": 6862, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def request_completion(prompt):

    completion_response = openai.chat.completions.create(
                            prompt=prompt,
                            temperature=0,
                            max_tokens=5,
                            top_p=1,
                            frequency_penalty=0,
                            presence_penalty=0,
                            model=COMPLETIONS_MODEL)

    return completion_response

def classify_transaction(transaction,prompt):

    prompt = prompt.replace('SUPPLIER_NAME',transaction['Supplier'])
    prompt = prompt.replace('DESCRIPTION_TEXT',transaction['Description'])
    prompt = prompt.replace('TRANSACTION_VALUE',str(transaction['Transaction value (£)']))

    classification = request_completion(prompt).choices[0].message.content.replace('\n','')

    return classification

# This function takes your training and validation outputs from the prepare_data function of the Finetuning API, and
# confirms that each have the same number of classes.
# If they do not have the same number of classes the fine-tune will fail and return an error

def check_finetune_classes(train_file,valid_file):

    train_classes = set()
    valid_classes = set()
    with open(train_file, 'r') as json_file:
        json_list = list(json_file)
        print(len(json_list))

    for json_str in json_list:
        result = json.loads(json_str)
        train_classes.add(result['completion'])
        #print(f"result: {result['completion']}")
        #print(isinstance(result, dict))

    with open(valid_file, 'r') as json_file:
        json_list = list(json_file)
        print(len(json_list))

    for json_str in json_list:
        result = json.loads(json_str)
        valid_classes.add(result['completion'])
        #print(f"result: {result['completion']}")
        #print(isinstance(result, dict))

    if len(train_classes) == len(valid_classes):
        print('All good')

    else:
        print('Classes do not match, please prepare data again')

```

## Zero-shot Classification

We'll first assess the performance of the base models at classifying these transactions using a simple prompt. We'll provide the model with 5 categories and a catch-all of "Could not classify" for ones that it cannot place.


```python
zero_shot_prompt = '''You are a data expert working for the National Library of Scotland.
You are analysing all transactions over £25,000 in value and classifying them into one of five categories.
The five categories are Building Improvement, Literature & Archive, Utility Bills, Professional Services and Software/IT.
If you can't tell what it is, say Could not classify

Transaction:

Supplier: SUPPLIER_NAME
Description: DESCRIPTION_TEXT
Value: TRANSACTION_VALUE

The classification is:'''

```


```python
# Get a test transaction
transaction = transactions.iloc[0]

# Interpolate the values into the prompt
prompt = zero_shot_prompt.replace('SUPPLIER_NAME',transaction['Supplier'])
prompt = prompt.replace('DESCRIPTION_TEXT',transaction['Description'])
prompt = prompt.replace('TRANSACTION_VALUE',str(transaction['Transaction value (£)']))

# Use our completion function to return a prediction
completion_response = request_completion(prompt)
print(completion_response.choices[0].text)

```

     Building Improvement


Our first attempt is correct, M & J Ballantyne Ltd are a house builder and the work they performed is indeed Building Improvement.

Lets expand the sample size to 25 and see how it performs, again with just a simple prompt to guide it


key: _node_content
string_value:
{"id_": "7652d4d8-4e52-4c40-93d4-09d2d9c2608d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0d33e960-07af-401f-ab60-1c9e37eec2d9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4a911dc26f5b7dc74dae948a37cb7e2aef7049301fd1064359dee0fd9b64d338", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f0eb2a92-9757-4854-ad5c-9c8489178282", "node_type": "1", "metadata": {}, "hash": "a7b37a180ec366ebae333ea01fbad729667d5e50d73da275472b863948435004", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5593, "end_char_idx": 8660, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
zero_shot_prompt = '''You are a data expert working for the National Library of Scotland.
You are analysing all transactions over £25,000 in value and classifying them into one of five categories.
The five categories are Building Improvement, Literature & Archive, Utility Bills, Professional Services and Software/IT.
If you can't tell what it is, say Could not classify

Transaction:

Supplier: SUPPLIER_NAME
Description: DESCRIPTION_TEXT
Value: TRANSACTION_VALUE

The classification is:'''

```


```python
# Get a test transaction
transaction = transactions.iloc[0]

# Interpolate the values into the prompt
prompt = zero_shot_prompt.replace('SUPPLIER_NAME',transaction['Supplier'])
prompt = prompt.replace('DESCRIPTION_TEXT',transaction['Description'])
prompt = prompt.replace('TRANSACTION_VALUE',str(transaction['Transaction value (£)']))

# Use our completion function to return a prediction
completion_response = request_completion(prompt)
print(completion_response.choices[0].text)

```

     Building Improvement


Our first attempt is correct, M & J Ballantyne Ltd are a house builder and the work they performed is indeed Building Improvement.

Lets expand the sample size to 25 and see how it performs, again with just a simple prompt to guide it


```python
test_transactions = transactions.iloc[:25]
test_transactions['Classification'] = test_transactions.apply(lambda x: classify_transaction(x,zero_shot_prompt),axis=1)

```

    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
    A value is trying to be set on a copy of a slice from a DataFrame.
    Try using .loc[row_indexer,col_indexer] = value instead
    
    See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
      



```python
test_transactions['Classification'].value_counts()

```




     Building Improvement    14
     Could not classify       5
     Literature & Archive     3
     Software/IT              2
     Utility Bills            1
    Name: Classification, dtype: int64




```python
test_transactions.head(25)

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>21/04/2016</td>
      <td>M &amp; J Ballantyne Ltd</td>
      <td>George IV Bridge Work</td>
      <td>35098.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>1</th>
      <td>26/04/2016</td>
      <td>Private Sale</td>
      <td>Literary &amp; Archival Items</td>
      <td>30000.0</td>
      <td>Literature &amp;


key: _node_content
string_value:
{"id_": "f0eb2a92-9757-4854-ad5c-9c8489178282", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7652d4d8-4e52-4c40-93d4-09d2d9c2608d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "d6d490b5ec6b605577082e7f64bfa33edf1a3ba144274e4f5501af94cea941f7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0b51c0e9-5088-4167-9545-84f9f903e605", "node_type": "1", "metadata": {}, "hash": "e03bb5f262906c04709166d5e2f9650f78379fcc51cd3487ee9e9e51e26e9813", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7466, "end_char_idx": 9598, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
test_transactions['Classification'].value_counts()

```




     Building Improvement    14
     Could not classify       5
     Literature & Archive     3
     Software/IT              2
     Utility Bills            1
    Name: Classification, dtype: int64




```python
test_transactions.head(25)

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>21/04/2016</td>
      <td>M &amp; J Ballantyne Ltd</td>
      <td>George IV Bridge Work</td>
      <td>35098.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>1</th>
      <td>26/04/2016</td>
      <td>Private Sale</td>
      <td>Literary &amp; Archival Items</td>
      <td>30000.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>2</th>
      <td>30/04/2016</td>
      <td>City Of Edinburgh Council</td>
      <td>Non Domestic Rates</td>
      <td>40800.0</td>
      <td>Utility Bills</td>
    </tr>
    <tr>
      <th>3</th>
      <td>09/05/2016</td>
      <td>Computacenter Uk</td>
      <td>Kelvin Hall</td>
      <td>72835.0</td>
      <td>Software/IT</td>
    </tr>
    <tr>
      <th>4</th>
      <td>09/05/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>64361.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>5</th>
      <td>09/05/2016</td>
      <td>A McGillivray</td>
      <td>Causewayside Refurbishment</td>
      <td>53690.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>6</th>
      <td>16/05/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>365344.


key: _node_content
string_value:
{"id_": "0b51c0e9-5088-4167-9545-84f9f903e605", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f0eb2a92-9757-4854-ad5c-9c8489178282", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "10b19c5bc0628e7b532c392f854cd3542b7abb3680def38b141869cab0c0ae74", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1ab2d71a-64f4-417d-bfef-a7891856493f", "node_type": "1", "metadata": {}, "hash": "830e162e27d2b6d8ca1907147fefe58cd0379410252231f0e057de4a8c4ebd14", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8591, "end_char_idx": 10519, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Archival Items</td>
      <td>30000.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>2</th>
      <td>30/04/2016</td>
      <td>City Of Edinburgh Council</td>
      <td>Non Domestic Rates</td>
      <td>40800.0</td>
      <td>Utility Bills</td>
    </tr>
    <tr>
      <th>3</th>
      <td>09/05/2016</td>
      <td>Computacenter Uk</td>
      <td>Kelvin Hall</td>
      <td>72835.0</td>
      <td>Software/IT</td>
    </tr>
    <tr>
      <th>4</th>
      <td>09/05/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>64361.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>5</th>
      <td>09/05/2016</td>
      <td>A McGillivray</td>
      <td>Causewayside Refurbishment</td>
      <td>53690.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>6</th>
      <td>16/05/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>365344.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>7</th>
      <td>23/05/2016</td>
      <td>Computacenter Uk</td>
      <td>Kelvin Hall</td>
      <td>26506.0</td>
      <td>Software/IT</td>
    </tr>
    <tr>
      <th>8</th>
      <td>23/05/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32777.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>9</th>
      <td>23/05/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32777.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>10</th>
      <td>30/05/2016</td>
      <td>ALDL</td>
      <td>ALDL Charges</td>
      <td>32317.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>11</th>
      <td>10/06/2016</td>
      <td>Wavetek Ltd</td>
      <td>Kelvin Hall</td>
      <td>87589.


key: _node_content
string_value:
{"id_": "1ab2d71a-64f4-417d-bfef-a7891856493f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0b51c0e9-5088-4167-9545-84f9f903e605", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9a2a01b0b97425a4a854c85abb984b66c40de45469d66038df4f600f3a0d9ffe", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9d86ee1f-116d-493e-a5f0-907891ccb160", "node_type": "1", "metadata": {}, "hash": "d4a6bc7aafe0d88c86c3a430b332d1d71e13a320791e9a2320d80249fe7d7968", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9598, "end_char_idx": 11500, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>7</th>
      <td>23/05/2016</td>
      <td>Computacenter Uk</td>
      <td>Kelvin Hall</td>
      <td>26506.0</td>
      <td>Software/IT</td>
    </tr>
    <tr>
      <th>8</th>
      <td>23/05/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32777.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>9</th>
      <td>23/05/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32777.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>10</th>
      <td>30/05/2016</td>
      <td>ALDL</td>
      <td>ALDL Charges</td>
      <td>32317.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>11</th>
      <td>10/06/2016</td>
      <td>Wavetek Ltd</td>
      <td>Kelvin Hall</td>
      <td>87589.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>12</th>
      <td>10/06/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>381803.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>13</th>
      <td>28/06/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32832.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>14</th>
      <td>30/06/2016</td>
      <td>Glasgow City Council</td>
      <td>Kelvin Hall</td>
      <td>1700000.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>15</th>
      <td>11/07/2016</td>
      <td>Wavetek Ltd</td>
      <td>Kelvin Hall</td>
      <td>65692.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>16</th>
      <td>11/07/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>139845.


key: _node_content
string_value:
{"id_": "9d86ee1f-116d-493e-a5f0-907891ccb160", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1ab2d71a-64f4-417d-bfef-a7891856493f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8dfcbf175329ef3fa9f490808280189d628a7a300804da39b1f3f55e0e94b915", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1b492b8c-1345-4515-b57c-9a8c780cdcdd", "node_type": "1", "metadata": {}, "hash": "aedecf401ca9f6e94fe7217bbd0d6a9a54911d173f0c08f2266b513dbc64b8f7", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10519, "end_char_idx": 12457, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>12</th>
      <td>10/06/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>381803.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>13</th>
      <td>28/06/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32832.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>14</th>
      <td>30/06/2016</td>
      <td>Glasgow City Council</td>
      <td>Kelvin Hall</td>
      <td>1700000.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>15</th>
      <td>11/07/2016</td>
      <td>Wavetek Ltd</td>
      <td>Kelvin Hall</td>
      <td>65692.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>16</th>
      <td>11/07/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>139845.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>17</th>
      <td>15/07/2016</td>
      <td>Sotheby'S</td>
      <td>Literary &amp; Archival Items</td>
      <td>28500.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>18</th>
      <td>18/07/2016</td>
      <td>Christies</td>
      <td>Literary &amp; Archival Items</td>
      <td>33800.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>19</th>
      <td>25/07/2016</td>
      <td>A McGillivray</td>
      <td>Causewayside Refurbishment</td>
      <td>30113.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>20</th>
      <td>31/07/2016</td>
      <td>ALDL</td>
      <td>ALDL Charges</td>
      <td>32317.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>21</th>
      <td>08/08/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32795.


key: _node_content
string_value:
{"id_": "1b492b8c-1345-4515-b57c-9a8c780cdcdd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9d86ee1f-116d-493e-a5f0-907891ccb160", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "13200a54ca549caf5804e7b6dbe2ddcf365b8971bce8858ef9c07bde1b0b636d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "75b67927-d868-4ad0-9792-90c2c3211562", "node_type": "1", "metadata": {}, "hash": "15145df5e444899d7eaa8215879e6dc395be1347aa4d7aefeb584951c647dbc3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11500, "end_char_idx": 13916, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>17</th>
      <td>15/07/2016</td>
      <td>Sotheby'S</td>
      <td>Literary &amp; Archival Items</td>
      <td>28500.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>18</th>
      <td>18/07/2016</td>
      <td>Christies</td>
      <td>Literary &amp; Archival Items</td>
      <td>33800.0</td>
      <td>Literature &amp; Archive</td>
    </tr>
    <tr>
      <th>19</th>
      <td>25/07/2016</td>
      <td>A McGillivray</td>
      <td>Causewayside Refurbishment</td>
      <td>30113.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>20</th>
      <td>31/07/2016</td>
      <td>ALDL</td>
      <td>ALDL Charges</td>
      <td>32317.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>21</th>
      <td>08/08/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32795.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>22</th>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>23</th>
      <td>15/08/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>196807.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>24</th>
      <td>24/08/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32795.0</td>
      <td>Building Improvement</td>
    </tr>
  </tbody>
</table>
</div>



Initial results are pretty good even with no labelled examples! The ones that it could not classify were tougher cases with few clues as to their topic, but maybe if we clean up the labelled dataset to give more examples we can get better performance.

## Classification with Embeddings

Lets create embeddings from the small set that we've classified so far - we've made a set of labelled examples by running the zero-shot classifier on 101 transactions from our dataset and manually correcting the 15 **Could not classify** results that we got

### Create embeddings

This initial section reuses the approach from the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb) to create embeddings from a combined field concatenating all of our features


key: _node_content
string_value:
{"id_": "75b67927-d868-4ad0-9792-90c2c3211562", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1b492b8c-1345-4515-b57c-9a8c780cdcdd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ce749810bc8cdacc4babaa1763208f7bf3ab04093b630162450245259d1642b4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "cd61d17f-248f-481d-a630-30a08dd9e50c", "node_type": "1", "metadata": {}, "hash": "592bf3fb5b250165f9be1287e5cbe1d0f913a07b87ff69149fa554370c6a15d1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12457, "end_char_idx": 13996, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>22</th>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866.0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>23</th>
      <td>15/08/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>196807.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>24</th>
      <td>24/08/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32795.0</td>
      <td>Building Improvement</td>
    </tr>
  </tbody>
</table>
</div>



Initial results are pretty good even with no labelled examples! The ones that it could not classify were tougher cases with few clues as to their topic, but maybe if we clean up the labelled dataset to give more examples we can get better performance.

## Classification with Embeddings

Lets create embeddings from the small set that we've classified so far - we've made a set of labelled examples by running the zero-shot classifier on 101 transactions from our dataset and manually correcting the 15 **Could not classify** results that we got

### Create embeddings

This initial section reuses the approach from the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb) to create embeddings from a combined field concatenating all of our features


```python
df = pd.read_csv('./data/labelled_transactions.csv')
df.head()

```


key: _node_content
string_value:
{"id_": "cd61d17f-248f-481d-a630-30a08dd9e50c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "75b67927-d868-4ad0-9792-90c2c3211562", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "927148b38f32cc552ad1c5b563a528f8a723c2d16c9dff46e19ba45c38a5f6eb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "677b0f6f-1fc9-48af-8d16-fe53afbbde0c", "node_type": "1", "metadata": {}, "hash": "889065215d9d9ee598b8c51a6044a6fbc48cb01e4bf38a2bfaf6bed16b6e41c4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12652, "end_char_idx": 15523, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Could not classify</td>
    </tr>
    <tr>
      <th>23</th>
      <td>15/08/2016</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>196807.0</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>24</th>
      <td>24/08/2016</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>32795.0</td>
      <td>Building Improvement</td>
    </tr>
  </tbody>
</table>
</div>



Initial results are pretty good even with no labelled examples! The ones that it could not classify were tougher cases with few clues as to their topic, but maybe if we clean up the labelled dataset to give more examples we can get better performance.

## Classification with Embeddings

Lets create embeddings from the small set that we've classified so far - we've made a set of labelled examples by running the zero-shot classifier on 101 transactions from our dataset and manually correcting the 15 **Could not classify** results that we got

### Create embeddings

This initial section reuses the approach from the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb) to create embeddings from a combined field concatenating all of our features


```python
df = pd.read_csv('./data/labelled_transactions.csv')
df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
    </tr>
    <tr>
      <th>1</th>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>2</th>
      <td>29/05/2017</td>
      <td>Morris &amp; Spottiswood Ltd</td>
      <td>George IV Bridge Work</td>
      <td>56448</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>3</th>
      <td>31/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>164691</td>
      <td>Building Improvement</td>
    </tr>
    <tr>
      <th>4</th>
      <td>24/07/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>27926</td>
      <td>Building Improvement</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "677b0f6f-1fc9-48af-8d16-fe53afbbde0c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "cd61d17f-248f-481d-a630-30a08dd9e50c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8e2a7a136dde499124b543b52cad23a6cbd54bfb427c34b86ef99c384ab73d5b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "82130577-447d-4c44-8e10-2cf7f5bf0861", "node_type": "1", "metadata": {}, "hash": "76cdbd68b0a7a9756c87f909fe93117379d8c683206dbf308bd92512c1a4d390", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15528, "end_char_idx": 18307, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
df['combined'] = "Supplier: " + df['Supplier'].str.strip() + "; Description: " + df['Description'].str.strip() + "; Value: " + str(df['Transaction value (£)']).strip()
df.head(2)

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc...</td>
    </tr>
    <tr>
      <th>1</th>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri...</td>
    </tr>
  </tbody>
</table>
</div>




```python
from transformers import GPT2TokenizerFast
tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")

df['n_tokens'] = df.combined.apply(lambda x: len(tokenizer.encode(x)))
len(df)

```




    101




```python
embedding_path = './data/transactions_with_embeddings_100.csv'

```


```python
from utils.embeddings_utils import get_embedding

df['babbage_similarity'] = df.combined.apply(lambda x: get_embedding(x, model='gpt-4'))
df['babbage_search'] = df.combined.apply(lambda x: get_embedding(x, model='gpt-4'))
df.to_csv(embedding_path)

```

### Use embeddings for classification

Now that we have our embeddings, let see if classifying these into the categories we've named gives us any more success.

For this we'll use a template from the [Classification_using_embeddings](Classification_using_embeddings.ipynb) notebook


```python
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, accuracy_score
from ast import literal_eval

fs_df = pd.read_csv(embedding_path)
fs_df["babbage_similarity"] = fs_df.babbage_similarity.apply(literal_eval).apply(np.array)
fs_df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;


key: _node_content
string_value:
{"id_": "82130577-447d-4c44-8e10-2cf7f5bf0861", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "677b0f6f-1fc9-48af-8d16-fe53afbbde0c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3243c8d9920301330bef1d1abafcf14c45947cdcb522cd33885a7b5a59fdb060", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "446b73e1-3106-4943-83de-0edc4ad09123", "node_type": "1", "metadata": {}, "hash": "db42d757381d67afc29cb12bebed20e8436ecba70894528e7e681938f9606eec", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
from transformers import GPT2TokenizerFast
tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")

df['n_tokens'] = df.combined.apply(lambda x: len(tokenizer.encode(x)))
len(df)

```




    101




```python
embedding_path = './data/transactions_with_embeddings_100.csv'

```


```python
from utils.embeddings_utils import get_embedding

df['babbage_similarity'] = df.combined.apply(lambda x: get_embedding(x, model='gpt-4'))
df['babbage_search'] = df.combined.apply(lambda x: get_embedding(x, model='gpt-4'))
df.to_csv(embedding_path)

```

### Use embeddings for classification

Now that we have our embeddings, let see if classifying these into the categories we've named gives us any more success.

For this we'll use a template from the [Classification_using_embeddings](Classification_using_embeddings.ipynb) notebook


```python
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, accuracy_score
from ast import literal_eval

fs_df = pd.read_csv(embedding_path)
fs_df["babbage_similarity"] = fs_df.babbage_similarity.apply(literal_eval).apply(np.array)
fs_df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Unnamed: 0</th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
      <th>n_tokens</th>
      <th>babbage_similarity</th>
      <th>babbage_search</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0</td>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
      <td>136</td>
      <td>[-0.009802100248634815, 0.022551486268639565, .</td>
      <td>[-0.00232666521333158, 0.019198870286345482, 0.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1</td>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692,


key: _node_content
string_value:
{"id_": "446b73e1-3106-4943-83de-0edc4ad09123", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "82130577-447d-4c44-8e10-2cf7f5bf0861", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "385e095a317422659747d5cfb9607b4f9b470264f3a9c360b0dd73c6af6305d6", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5a30e65b-efb9-4901-983c-6fdd0380792c", "node_type": "1", "metadata": {}, "hash": "75eb4af279ae92b39b3f9d54677b49f533165059011eb145c7316546cf203d63", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Desc.</td>
      <td>136</td>
      <td>[-0.009802100248634815, 0.022551486268639565, .</td>
      <td>[-0.00232666521333158, 0.019198870286345482, 0.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1</td>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2</td>
      <td>29/05/2017</td>
      <td>Morris &amp; Spottiswood Ltd</td>
      <td>George IV Bridge Work</td>
      <td>56448</td>
      <td>Building Improvement</td>
      <td>Supplier: Morris &amp; Spottiswood Ltd; Descriptio.</td>
      <td>141</td>
      <td>[-0.009000026620924473, 0.02405017428100109, -.</td>
      <td>[0.0028343256562948227, 0.021166473627090454, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>3</td>
      <td>31/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>164691</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
    <tr>
      <th>4</th>
      <td>4</td>
      <td>24/07/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>27926</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "5a30e65b-efb9-4901-983c-6fdd0380792c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "446b73e1-3106-4943-83de-0edc4ad09123", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "527283567c29836edac2fe39e40e6833e7652d6325578000ec786e80049c540f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c0106133-f6a1-4199-9f80-6745d4bdd64a", "node_type": "1", "metadata": {}, "hash": "6ef9c34db62ad67a5062f7d044a001630949335a8deb9d41f006656b4a4fa7bf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
009000026620924473, 0.02405017428100109, -.</td>
      <td>[0.0028343256562948227, 0.021166473627090454, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>3</td>
      <td>31/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>164691</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
    <tr>
      <th>4</th>
      <td>4</td>
      <td>24/07/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>27926</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
  </tbody>
</table>
</div>




```python
X_train, X_test, y_train, y_test = train_test_split(
    list(fs_df.babbage_similarity.values), fs_df.Classification, test_size=0.2, random_state=42
)

clf = RandomForestClassifier(n_estimators=100)
clf.fit(X_train, y_train)
preds = clf.predict(X_test)
probas = clf.predict_proba(X_test)

report = classification_report(y_test, preds)
print(report)

```

                          precision    recall  f1-score   support
    
    Building Improvement       0.92      1.00      0.96        11
    Literature & Archive       1.00      1.00      1.00         3
                   Other       0.00      0.00      0.00         1
             Software/IT       1.00      1.00      1.00         1
           Utility Bills       1.00      1.00      1.00         5
    
                accuracy                           0.95        21
               macro avg       0.78      0.80      0.79        21
            weighted avg       0.91      0.95      0.93        21


key: _node_content
string_value:
{"id_": "c0106133-f6a1-4199-9f80-6745d4bdd64a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5a30e65b-efb9-4901-983c-6fdd0380792c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "541347794e57bce196d12e5787e8dadcfcaad85730fcf2bcbea13f85dd9e0821", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "833bbade-d795-408d-b93c-2f164234ffa5", "node_type": "1", "metadata": {}, "hash": "97646124049729c9fb7cf68320ca735c460657c15f246b2aead230f80708b6c3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Descri.</td>
      <td>140</td>
      <td>[-0.009065819904208183, 0.012094118632376194, .</td>
      <td>[0.005169447045773268, 0.00473341578617692, -0.</td>
    </tr>
  </tbody>
</table>
</div>




```python
X_train, X_test, y_train, y_test = train_test_split(
    list(fs_df.babbage_similarity.values), fs_df.Classification, test_size=0.2, random_state=42
)

clf = RandomForestClassifier(n_estimators=100)
clf.fit(X_train, y_train)
preds = clf.predict(X_test)
probas = clf.predict_proba(X_test)

report = classification_report(y_test, preds)
print(report)

```

                          precision    recall  f1-score   support
    
    Building Improvement       0.92      1.00      0.96        11
    Literature & Archive       1.00      1.00      1.00         3
                   Other       0.00      0.00      0.00         1
             Software/IT       1.00      1.00      1.00         1
           Utility Bills       1.00      1.00      1.00         5
    
                accuracy                           0.95        21
               macro avg       0.78      0.80      0.79        21
            weighted avg       0.91      0.95      0.93        21
    


    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))
    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))
    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))


key: _node_content
string_value:
{"id_": "833bbade-d795-408d-b93c-2f164234ffa5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c0106133-f6a1-4199-9f80-6745d4bdd64a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "894391993c298de19d932835af4971da8deb75db582ec3d6d30b4df80cabef0f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d061c796-e701-4024-949f-f4db590ccb9b", "node_type": "1", "metadata": {}, "hash": "8ef642a7ef9308e635ac9210416a2232381f337cbe2b9257ec79078e95443c7e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 21796, "end_char_idx": 24608, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))
    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))
    /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
      _warn_prf(average, modifier, msg_start, len(result))


Performance for this model is pretty strong, so creating embeddings and using even a simpler classifier looks like an effective approach as well, with the zero-shot classifier helping us do the initial classification of the unlabelled dataset.

Lets take it one step further and see if a fine-tuned model trained on this same labelled datasets gives us comparable results

## Fine-tuned Transaction Classification

For this use case we're going to try to improve on the few-shot classification from above by training a fine-tuned model on the same labelled set of 101 transactions and applying this fine-tuned model on group of unseen transactions

### Building Fine-tuned Classifier

We'll need to do some data prep first to get our data ready. This will take the following steps:
- First we'll list out our classes and replace them with numeric identifiers. Making the model predict a single token rather than multiple consecutive ones like 'Building Improvement' should give us better results
- We also need to add a common prefix and suffix to each example to aid the model in making predictions - in our case our text is already started with 'Supplier' and we'll add a suffix of '\n\n###\n\n'
- Lastly we'll aid a leading whitespace onto each of our target classes for classification, again to aid the model


```python
ft_prep_df = fs_df.copy()
len(ft_prep_df)

```




    101




```python
ft_prep_df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;


key: _node_content
string_value:
{"id_": "d061c796-e701-4024-949f-f4db590ccb9b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "833bbade-d795-408d-b93c-2f164234ffa5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "1c7f1b5cffc7d7ab81e5d6010bc82e0ce128d8ce4f670af63bf4b52f87cfe4a7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "41c1c376-4ec9-48fb-a34b-bbef9b5ab513", "node_type": "1", "metadata": {}, "hash": "a5eb33482f5beab940c70a56ab460e043a33f7e25d64608836444cd45fc2f9a3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Performance for this model is pretty strong, so creating embeddings and using even a simpler classifier looks like an effective approach as well, with the zero-shot classifier helping us do the initial classification of the unlabelled dataset.

Lets take it one step further and see if a fine-tuned model trained on this same labelled datasets gives us comparable results

## Fine-tuned Transaction Classification

For this use case we're going to try to improve on the few-shot classification from above by training a fine-tuned model on the same labelled set of 101 transactions and applying this fine-tuned model on group of unseen transactions

### Building Fine-tuned Classifier

We'll need to do some data prep first to get our data ready. This will take the following steps:
- First we'll list out our classes and replace them with numeric identifiers. Making the model predict a single token rather than multiple consecutive ones like 'Building Improvement' should give us better results
- We also need to add a common prefix and suffix to each example to aid the model in making predictions - in our case our text is already started with 'Supplier' and we'll add a suffix of '\n\n###\n\n'
- Lastly we'll aid a leading whitespace onto each of our target classes for classification, again to aid the model


```python
ft_prep_df = fs_df.copy()
len(ft_prep_df)

```




    101




```python
ft_prep_df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Unnamed: 0</th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
      <th>n_tokens</th>
      <th>babbage_similarity</th>
      <th>babbage_search</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0</td>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
      <td>12</td>
      <td>[-0.009630300104618073, 0.009887108579277992, .</td>
      <td>[-0.008217384107410908, 0.025170527398586273, .</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1</td>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.


key: _node_content
string_value:
{"id_": "41c1c376-4ec9-48fb-a34b-bbef9b5ab513", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d061c796-e701-4024-949f-f4db590ccb9b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b10bdc2b34ed3eef42c320c4ea575c1c3dd863c1cb26a49560224515fb0ccfa2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "12cad3fb-11fa-4691-b6cf-2b16f6a85517", "node_type": "1", "metadata": {}, "hash": "1bec2b60456b320bcd176e95eaeb80aaace271e161f85c583d9bc8305a7c123f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
">
      <th></th>
      <th>Unnamed: 0</th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
      <th>n_tokens</th>
      <th>babbage_similarity</th>
      <th>babbage_search</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0</td>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
      <td>12</td>
      <td>[-0.009630300104618073, 0.009887108579277992, .</td>
      <td>[-0.008217384107410908, 0.025170527398586273, .</td>
    </tr>
    <tr>
      <th>1</th>
      <td>1</td>
      <td>29/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>74806</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2</td>
      <td>29/05/2017</td>
      <td>Morris &amp; Spottiswood Ltd</td>
      <td>George IV Bridge Work</td>
      <td>56448</td>
      <td>Building Improvement</td>
      <td>Supplier: Morris &amp; Spottiswood Ltd; Descriptio.</td>
      <td>17</td>
      <td>[-0.005225738976150751, 0.015156379900872707, .</td>
      <td>[-0.007611643522977829, 0.030322374776005745, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>3</td>
      <td>31/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>164691</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .


key: _node_content
string_value:
{"id_": "12cad3fb-11fa-4691-b6cf-2b16f6a85517", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "41c1c376-4ec9-48fb-a34b-bbef9b5ab513", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f66e8505ea5557fc48a57ec25b8af3674b972cf4d466dfaa7e1fee094e58efba", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "99386c05-d003-4991-9fef-0a09201ba091", "node_type": "1", "metadata": {}, "hash": "add07faf72b3a64659972d103efe126f1757910fa8db3e531a9efee7985e2e3a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2</td>
      <td>29/05/2017</td>
      <td>Morris &amp; Spottiswood Ltd</td>
      <td>George IV Bridge Work</td>
      <td>56448</td>
      <td>Building Improvement</td>
      <td>Supplier: Morris &amp; Spottiswood Ltd; Descriptio.</td>
      <td>17</td>
      <td>[-0.005225738976150751, 0.015156379900872707, .</td>
      <td>[-0.007611643522977829, 0.030322374776005745, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>3</td>
      <td>31/05/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>164691</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
    <tr>
      <th>4</th>
      <td>4</td>
      <td>24/07/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>27926</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
  </tbody>
</table>
</div>




```python
classes = list(set(ft_prep_df['Classification']))
class_df = pd.DataFrame(classes).reset_index()
class_df.columns = ['class_id','class']
class_df  , len(class_df)

```




    (   class_id                 class
     0         0  Literature & Archive
     1         1         Utility Bills
     2         2  Building Improvement
     3         3           Software/IT
     4         4                 Other,
     5)


key: _node_content
string_value:
{"id_": "99386c05-d003-4991-9fef-0a09201ba091", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "12cad3fb-11fa-4691-b6cf-2b16f6a85517", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "188db50a49e7a7946b2d118d0aceb8e97544135f0fb41a0b3a0f7175d60d73aa", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5d089d98-881d-47b7-a76d-3029b4241ed9", "node_type": "1", "metadata": {}, "hash": "d3e5fd22f04e9ead2e8092cb0b2f36a08c6b12f7e41382d4a69109fc7f935b31", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
    <tr>
      <th>4</th>
      <td>4</td>
      <td>24/07/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>27926</td>
      <td>Building Improvement</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>16</td>
      <td>[-0.006144719664007425, -0.0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
  </tbody>
</table>
</div>




```python
classes = list(set(ft_prep_df['Classification']))
class_df = pd.DataFrame(classes).reset_index()
class_df.columns = ['class_id','class']
class_df  , len(class_df)

```




    (   class_id                 class
     0         0  Literature & Archive
     1         1         Utility Bills
     2         2  Building Improvement
     3         3           Software/IT
     4         4                 Other,
     5)




```python
ft_df_with_class = ft_prep_df.merge(class_df,left_on='Classification',right_on='class',how='inner')

# Adding a leading whitespace onto each completion to help the model
ft_df_with_class['class_id'] = ft_df_with_class.apply(lambda x: ' ' + str(x['class_id']),axis=1)
ft_df_with_class = ft_df_with_class.drop('class', axis=1)

# Adding a common separator onto the end of each prompt so the model knows when a prompt is terminating
ft_df_with_class['prompt'] = ft_df_with_class.apply(lambda x: x['combined'] + '\n\n###\n\n',axis=1)
ft_df_with_class.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;


key: _node_content
string_value:
{"id_": "5d089d98-881d-47b7-a76d-3029b4241ed9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "99386c05-d003-4991-9fef-0a09201ba091", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "5a3c6f1c72c50d7e12949bc0a02e4b90a9e2759d8c34d6976a73d186f792647b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5e971462-d072-43b0-9f47-d411746f0e2a", "node_type": "1", "metadata": {}, "hash": "d56e503d28f89403ccf236869a869cc4844716596bc4cd70b39701b713a9230b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0018709596479311585.</td>
      <td>[-0.007424891460686922, 0.008475713431835175, .</td>
    </tr>
  </tbody>
</table>
</div>




```python
classes = list(set(ft_prep_df['Classification']))
class_df = pd.DataFrame(classes).reset_index()
class_df.columns = ['class_id','class']
class_df  , len(class_df)

```




    (   class_id                 class
     0         0  Literature & Archive
     1         1         Utility Bills
     2         2  Building Improvement
     3         3           Software/IT
     4         4                 Other,
     5)




```python
ft_df_with_class = ft_prep_df.merge(class_df,left_on='Classification',right_on='class',how='inner')

# Adding a leading whitespace onto each completion to help the model
ft_df_with_class['class_id'] = ft_df_with_class.apply(lambda x: ' ' + str(x['class_id']),axis=1)
ft_df_with_class = ft_df_with_class.drop('class', axis=1)

# Adding a common separator onto the end of each prompt so the model knows when a prompt is terminating
ft_df_with_class['prompt'] = ft_df_with_class.apply(lambda x: x['combined'] + '\n\n###\n\n',axis=1)
ft_df_with_class.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Unnamed: 0</th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
      <th>n_tokens</th>
      <th>babbage_similarity</th>
      <th>babbage_search</th>
      <th>class_id</th>
      <th>prompt</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0</td>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
      <td>12</td>
      <td>[-0.009630300104618073, 0.009887108579277992, .</td>
      <td>[-0.008217384107410908, 0.025170527398586273, .</td>
      <td>4</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.


key: _node_content
string_value:
{"id_": "5e971462-d072-43b0-9f47-d411746f0e2a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5d089d98-881d-47b7-a76d-3029b4241ed9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c12e40b85cbabe85e4fd67cc70e6fd033f0e1fc1d9466fb9c3590c10a23e9d01", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8a9533d4-d349-4ead-82db-648fc5abad81", "node_type": "1", "metadata": {}, "hash": "f572f84beb4b5b472763e4c6816fc6ab10e397cc5a7600dd6f69a1a23e231c4a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Unnamed: 0</th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>Classification</th>
      <th>combined</th>
      <th>n_tokens</th>
      <th>babbage_similarity</th>
      <th>babbage_search</th>
      <th>class_id</th>
      <th>prompt</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0</td>
      <td>15/08/2016</td>
      <td>Creative Video Productions Ltd</td>
      <td>Kelvin Hall</td>
      <td>26866</td>
      <td>Other</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
      <td>12</td>
      <td>[-0.009630300104618073, 0.009887108579277992, .</td>
      <td>[-0.008217384107410908, 0.025170527398586273, .</td>
      <td>4</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>51</td>
      <td>31/03/2017</td>
      <td>NLS Foundation</td>
      <td>Grant Payment</td>
      <td>177500</td>
      <td>Other</td>
      <td>Supplier: NLS Foundation; Description: Grant P.</td>
      <td>11</td>
      <td>[-0.022305507212877274, 0.008543581701815128, .</td>
      <td>[-0.020519884303212166, 0.01993306167423725, -.</td>
      <td>4</td>
      <td>Supplier: NLS Foundation; Description: Grant P.</td>
    </tr>
    <tr>
      <th>2</th>
      <td>70</td>
      <td>26/06/2017</td>
      <td>British Library</td>
      <td>Legal Deposit Services</td>
      <td>50056</td>
      <td>Other</td>
      <td>Supplier: British Library; Description: Legal .</td>
      <td>11</td>
      <td>[-0.01019938476383686, 0.015277703292667866, -.</td>
      <td>[-0.01843327097594738, 0.03343546763062477, -0.</td>
      <td>4</td>
      <td>Supplier: British Library; Description: Legal .


key: _node_content
string_value:
{"id_": "8a9533d4-d349-4ead-82db-648fc5abad81", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5e971462-d072-43b0-9f47-d411746f0e2a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ed12dcc9e9fd608d5a13874b76f3081cdc72046900f609aac1a183123a9aee45", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "73522ee7-2178-4093-a1b9-fb6edc7a39a1", "node_type": "1", "metadata": {}, "hash": "a88dd24c19611bf1c2d00df8509e7ac2f1b1745a07af31f18402b795d134369f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0.025170527398586273, .</td>
      <td>4</td>
      <td>Supplier: Creative Video Productions Ltd; Desc.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>51</td>
      <td>31/03/2017</td>
      <td>NLS Foundation</td>
      <td>Grant Payment</td>
      <td>177500</td>
      <td>Other</td>
      <td>Supplier: NLS Foundation; Description: Grant P.</td>
      <td>11</td>
      <td>[-0.022305507212877274, 0.008543581701815128, .</td>
      <td>[-0.020519884303212166, 0.01993306167423725, -.</td>
      <td>4</td>
      <td>Supplier: NLS Foundation; Description: Grant P.</td>
    </tr>
    <tr>
      <th>2</th>
      <td>70</td>
      <td>26/06/2017</td>
      <td>British Library</td>
      <td>Legal Deposit Services</td>
      <td>50056</td>
      <td>Other</td>
      <td>Supplier: British Library; Description: Legal .</td>
      <td>11</td>
      <td>[-0.01019938476383686, 0.015277703292667866, -.</td>
      <td>[-0.01843327097594738, 0.03343546763062477, -0.</td>
      <td>4</td>
      <td>Supplier: British Library; Description: Legal .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>71</td>
      <td>24/07/2017</td>
      <td>ALDL</td>
      <td>Legal Deposit Services</td>
      <td>27067</td>
      <td>Other</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
      <td>11</td>
      <td>[-0.008471488021314144, 0.004098685923963785, .</td>
      <td>[-0.012966590002179146, 0.01299362163990736, 0.</td>
      <td>4</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
    </tr>
    <tr>
      <th>4</th>
      <td>100</td>
      <td>24/07/2017</td>
      <td>AM Phillip</td>
      <td>Vehicle Purchase</td>
      <td>26604</td>
      <td>Other</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
      <td>10</td>
      <td>[-0.003459023078903556, 0.004626389592885971, .</td>
      <td>[-0.0010945454705506563, 0.008626140654087067,</td>
      <td>4</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "73522ee7-2178-4093-a1b9-fb6edc7a39a1", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8a9533d4-d349-4ead-82db-648fc5abad81", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "47ca01123456015dd7452a0dd94814d4ec65dfa3c3124483ac5ba24701a8a716", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c29401fe-b204-4fb1-851e-4cf11ec7dab8", "node_type": "1", "metadata": {}, "hash": "17aea8b93e124beffc599e6753771726f91f1475957e5af92c544a223902196e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Description: Legal .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>71</td>
      <td>24/07/2017</td>
      <td>ALDL</td>
      <td>Legal Deposit Services</td>
      <td>27067</td>
      <td>Other</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
      <td>11</td>
      <td>[-0.008471488021314144, 0.004098685923963785, .</td>
      <td>[-0.012966590002179146, 0.01299362163990736, 0.</td>
      <td>4</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
    </tr>
    <tr>
      <th>4</th>
      <td>100</td>
      <td>24/07/2017</td>
      <td>AM Phillip</td>
      <td>Vehicle Purchase</td>
      <td>26604</td>
      <td>Other</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
      <td>10</td>
      <td>[-0.003459023078903556, 0.004626389592885971, .</td>
      <td>[-0.0010945454705506563, 0.008626140654087067,</td>
      <td>4</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
    </tr>
  </tbody>
</table>
</div>




```python
# This step is unnecessary if you have a number of observations in each class
# In our case we don't, so we shuffle the data to give us a better chance of getting equal classes in our train and validation sets
# Our fine-tuned model will error if we have less classes in the validation set, so this is a necessary step

import random

labels = [x for x in ft_df_with_class['class_id']]
text = [x for x in ft_df_with_class['prompt']]
ft_df = pd.DataFrame(zip(text, labels), columns = ['prompt','class_id']) #[:300]
ft_df.columns = ['prompt','completion']
ft_df['ordering'] = ft_df.apply(lambda x: random.randint(0,len(ft_df)), axis = 1)
ft_df.set_index('ordering',inplace=True)
ft_df_sorted = ft_df.sort_index(ascending=True)
ft_df_sorted.head()

```


key: _node_content
string_value:
{"id_": "c29401fe-b204-4fb1-851e-4cf11ec7dab8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "73522ee7-2178-4093-a1b9-fb6edc7a39a1", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a80e306b3df2ae00559cc4f0ff6f8531a6f6b0d60df6d8cdd7c899251f03937e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e4327d1f-5196-47e0-9c8a-1b8d9521ef33", "node_type": "1", "metadata": {}, "hash": "04d0ab528b28e65630594e853a00373b35e2f74cb6eb526846098ddd88a472f1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Description: Legal Deposit Ser.</td>
    </tr>
    <tr>
      <th>4</th>
      <td>100</td>
      <td>24/07/2017</td>
      <td>AM Phillip</td>
      <td>Vehicle Purchase</td>
      <td>26604</td>
      <td>Other</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
      <td>10</td>
      <td>[-0.003459023078903556, 0.004626389592885971, .</td>
      <td>[-0.0010945454705506563, 0.008626140654087067,</td>
      <td>4</td>
      <td>Supplier: AM Phillip; Description: Vehicle Pur.</td>
    </tr>
  </tbody>
</table>
</div>




```python
# This step is unnecessary if you have a number of observations in each class
# In our case we don't, so we shuffle the data to give us a better chance of getting equal classes in our train and validation sets
# Our fine-tuned model will error if we have less classes in the validation set, so this is a necessary step

import random

labels = [x for x in ft_df_with_class['class_id']]
text = [x for x in ft_df_with_class['prompt']]
ft_df = pd.DataFrame(zip(text, labels), columns = ['prompt','class_id']) #[:300]
ft_df.columns = ['prompt','completion']
ft_df['ordering'] = ft_df.apply(lambda x: random.randint(0,len(ft_df)), axis = 1)
ft_df.set_index('ordering',inplace=True)
ft_df_sorted = ft_df.sort_index(ascending=True)
ft_df_sorted.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>prompt</th>
      <th>completion</th>
    </tr>
    <tr>
      <th>ordering</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Supplier: Sothebys; Description: Literary &amp; Ar...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Supplier: Sotheby'S; Description: Literary &amp; A...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: City Of Edinburgh Council; Descripti...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: John Graham Construction Ltd; Descri...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Supplier: John Graham Construction Ltd; Descri...</td>
      <td>2</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "e4327d1f-5196-47e0-9c8a-1b8d9521ef33", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c29401fe-b204-4fb1-851e-4cf11ec7dab8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "59e61a323af642e2eba58b5fccbeb9aeffb37da897a87f152a6f305f3e66dd75", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e3002c28-cf15-47dd-97b5-510c19fc160b", "node_type": "1", "metadata": {}, "hash": "5f3b6d6e7de69457a831ae7d4876ac4b68e05d81656713c91ad9ed6c18589e94", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 31999, "end_char_idx": 34657, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>prompt</th>
      <th>completion</th>
    </tr>
    <tr>
      <th>ordering</th>
      <th></th>
      <th></th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Supplier: Sothebys; Description: Literary &amp; Ar...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Supplier: Sotheby'S; Description: Literary &amp; A...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: City Of Edinburgh Council; Descripti...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: John Graham Construction Ltd; Descri...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Supplier: John Graham Construction Ltd; Descri...</td>
      <td>2</td>
    </tr>
  </tbody>
</table>
</div>




```python
# This step is to remove any existing files if we've already produced training/validation sets for this classifier
#!rm transactions_grouped*

# We output our shuffled dataframe to a .jsonl file and run the prepare_data function to get us our input files
ft_df_sorted.to_json("transactions_grouped.jsonl", orient='records', lines=True)
!openai tools fine_tunes.prepare_data -f transactions_grouped.jsonl -q

```


```python
# This functions checks that your classes all appear in both prepared files
# If they don't, the fine-tuned model creation will fail
check_finetune_classes('transactions_grouped_prepared_train.jsonl','transactions_grouped_prepared_valid.jsonl')

```

    31
    8
    All good



```python
# This step creates your model
!openai api fine_tunes.create -t "transactions_grouped_prepared_train.jsonl" -v "transactions_grouped_prepared_valid.jsonl" --compute_classification_metrics --classification_n_classes 5 -m curie

# You can use following command to get fine tuning job status and model name, replace the job name with your job
#!openai api fine_tunes.get -i ft-YBIc01t4hxYBC7I5qhRF3Qdx

```


```python
# Congrats, you've got a fine-tuned model!
# Copy/paste the name provided into the variable below and we'll take it for a spin
fine_tuned_model = 'curie:ft-personal-2022-10-20-10-42-56'

```

### Applying Fine-tuned Classifier

Now we'll apply our classifier to see how it performs. We only had 31 unique observations in our training set and 8 in our validation set, so lets see how the performance is


key: _node_content
string_value:
{"id_": "e3002c28-cf15-47dd-97b5-510c19fc160b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e4327d1f-5196-47e0-9c8a-1b8d9521ef33", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "01d50bcd5da1661829a9ba762aa605e266249b4093f03944f62c9e75bebce4a2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "96a6defe-3c3c-4306-a009-f38b1bd45be5", "node_type": "1", "metadata": {}, "hash": "87bc6eaa26421b3f582a0c7dc06569d5063bf5cb0c22ba384959e1a8a72586fa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 33116, "end_char_idx": 35809, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# This step is to remove any existing files if we've already produced training/validation sets for this classifier
#!rm transactions_grouped*

# We output our shuffled dataframe to a .jsonl file and run the prepare_data function to get us our input files
ft_df_sorted.to_json("transactions_grouped.jsonl", orient='records', lines=True)
!openai tools fine_tunes.prepare_data -f transactions_grouped.jsonl -q

```


```python
# This functions checks that your classes all appear in both prepared files
# If they don't, the fine-tuned model creation will fail
check_finetune_classes('transactions_grouped_prepared_train.jsonl','transactions_grouped_prepared_valid.jsonl')

```

    31
    8
    All good



```python
# This step creates your model
!openai api fine_tunes.create -t "transactions_grouped_prepared_train.jsonl" -v "transactions_grouped_prepared_valid.jsonl" --compute_classification_metrics --classification_n_classes 5 -m curie

# You can use following command to get fine tuning job status and model name, replace the job name with your job
#!openai api fine_tunes.get -i ft-YBIc01t4hxYBC7I5qhRF3Qdx

```


```python
# Congrats, you've got a fine-tuned model!
# Copy/paste the name provided into the variable below and we'll take it for a spin
fine_tuned_model = 'curie:ft-personal-2022-10-20-10-42-56'

```

### Applying Fine-tuned Classifier

Now we'll apply our classifier to see how it performs. We only had 31 unique observations in our training set and 8 in our validation set, so lets see how the performance is


```python
test_set = pd.read_json('transactions_grouped_prepared_valid.jsonl', lines=True)
test_set.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>prompt</th>
      <th>completion</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Supplier: Wavetek Ltd; Description: Kelvin Hal...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Supplier: ECG Facilities Service; Description:...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: M &amp; J Ballantyne Ltd; Description: G...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Supplier: Private Sale; Description: Literary ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Supplier: Ex Libris; Description: IT equipment...</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "96a6defe-3c3c-4306-a009-f38b1bd45be5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e3002c28-cf15-47dd-97b5-510c19fc160b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "346af45a07027d149689cbde56d171aad772eac2c2dac6b1c6b977fea84b32e3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "321e0291-256c-4724-8b4d-d6c230610f67", "node_type": "1", "metadata": {}, "hash": "4c99148838b6b191df762433964413bb76ad874e19bb1eb5bdbc4ede49b911ba", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 34660, "end_char_idx": 36863, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
test_set = pd.read_json('transactions_grouped_prepared_valid.jsonl', lines=True)
test_set.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>prompt</th>
      <th>completion</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Supplier: Wavetek Ltd; Description: Kelvin Hal...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Supplier: ECG Facilities Service; Description:...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Supplier: M &amp; J Ballantyne Ltd; Description: G...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Supplier: Private Sale; Description: Literary ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Supplier: Ex Libris; Description: IT equipment...</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
</div>




```python
test_set['predicted_class'] = test_set.apply(lambda x: openai.chat.completions.create(model=fine_tuned_model, prompt=x['prompt'], max_tokens=1, temperature=0, logprobs=5),axis=1)
test_set['pred'] = test_set.apply(lambda x : x['predicted_class']['choices'][0]['text'],axis=1)

```


```python
test_set['result'] = test_set.apply(lambda x: str(x['pred']).strip() == str(x['completion']).strip(), axis = 1)

```


```python
test_set['result'].value_counts()

```




    True     4
    False    4
    Name: result, dtype: int64



Performance is not great - unfortunately this is expected. With only a few examples of each class, the above approach with embeddings and a traditional classifier worked better.

A fine-tuned model works best with a great number of labelled observations. If we had a few hundred or thousand we may get better results, but lets do one last test on a holdout set to confirm that it doesn't generalise well to a new set of observations


```python
holdout_df = transactions.copy().iloc[101:]
holdout_df.head()

```


key: _node_content
string_value:
{"id_": "321e0291-256c-4724-8b4d-d6c230610f67", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "96a6defe-3c3c-4306-a009-f38b1bd45be5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9312c5a5059ef63590485ded5694e5c1161e037162f5078440f90361dbe6ea2e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "073f1699-9307-4fa3-aceb-6d0b11ce5094", "node_type": "1", "metadata": {}, "hash": "41384319033ee828ca1213e14ae9187526ad5e9d62c05aefaafb93c1a0300722", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 35814, "end_char_idx": 38186, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
test_set['predicted_class'] = test_set.apply(lambda x: openai.chat.completions.create(model=fine_tuned_model, prompt=x['prompt'], max_tokens=1, temperature=0, logprobs=5),axis=1)
test_set['pred'] = test_set.apply(lambda x : x['predicted_class']['choices'][0]['text'],axis=1)

```


```python
test_set['result'] = test_set.apply(lambda x: str(x['pred']).strip() == str(x['completion']).strip(), axis = 1)

```


```python
test_set['result'].value_counts()

```




    True     4
    False    4
    Name: result, dtype: int64



Performance is not great - unfortunately this is expected. With only a few examples of each class, the above approach with embeddings and a traditional classifier worked better.

A fine-tuned model works best with a great number of labelled observations. If we had a few hundred or thousand we may get better results, but lets do one last test on a holdout set to confirm that it doesn't generalise well to a new set of observations


```python
holdout_df = transactions.copy().iloc[101:]
holdout_df.head()

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>101</th>
      <td>23/10/2017</td>
      <td>City Building LLP</td>
      <td>Causewayside Refurbishment</td>
      <td>53147.0</td>
    </tr>
    <tr>
      <th>102</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
    </tr>
    <tr>
      <th>103</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
    </tr>
    <tr>
      <th>104</th>
      <td>06/11/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>134208.0</td>
    </tr>
    <tr>
      <th>105</th>
      <td>06/11/2017</td>
      <td>ALDL</td>
      <td>Legal Deposit Services</td>
      <td>27067.0</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "073f1699-9307-4fa3-aceb-6d0b11ce5094", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "321e0291-256c-4724-8b4d-d6c230610f67", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0748581bbd1585b76c4f8c1c34f6acb7d72db21b2e140dc7bb7838a7fe2d4dff", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "eb1b193e-bef8-428c-8e2b-250b4add5e7f", "node_type": "1", "metadata": {}, "hash": "fe4f86c66fbbb9ef72a8997e5840c020cd24c34213c813d4833ba41536501dcc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
holdout_df['combined'] = "Supplier: " + holdout_df['Supplier'].str.strip() + "; Description: " + holdout_df['Description'].str.strip() + '\n\n###\n\n' # + "; Value: " + str(df['Transaction value (£)']).strip()
holdout_df['prediction_result'] = holdout_df.apply(lambda x: openai.chat.completions.create(model=fine_tuned_model, prompt=x['combined'], max_tokens=1, temperature=0, logprobs=5),axis=1)
holdout_df['pred'] = holdout_df.apply(lambda x : x['prediction_result']['choices'][0]['text'],axis=1)

```


```python
holdout_df.head(10)

```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Date</th>
      <th>Supplier</th>
      <th>Description</th>
      <th>Transaction value (£)</th>
      <th>combined</th>
      <th>prediction_result</th>
      <th>pred</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>101</th>
      <td>23/10/2017</td>
      <td>City Building LLP</td>
      <td>Causewayside Refurbishment</td>
      <td>53147.0</td>
      <td>Supplier: City Building LLP; Description: Caus.</td>
      <td>{'id': 'cmpl-63YDadbYLo8xKsGY2vReOFCMgTOvG', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>102</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDbNK1D7UikDc3xi5ATihg5kQEt', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>103</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDbwfiHjkjMWsfTKNt6naeqPzOe', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>104</th>
      <td>06/11/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>134208.0</td>
      <td>Supplier: John Graham Construction Ltd; Descri.


key: _node_content
string_value:
{"id_": "eb1b193e-bef8-428c-8e2b-250b4add5e7f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "073f1699-9307-4fa3-aceb-6d0b11ce5094", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "7a1ce940607e9f8ab5257719d5e347e11c48527f002e1e54b5fcaa5bb3d53a1f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "64729878-a8ef-4c2c-90c2-aff27fc9a8e1", "node_type": "1", "metadata": {}, "hash": "d831bf9d31817458fcca86434785cd9812cf4bd6d25bb8f6d695f78c1d469424", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Supplier: City Building LLP; Description: Caus.</td>
      <td>{'id': 'cmpl-63YDadbYLo8xKsGY2vReOFCMgTOvG', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>102</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDbNK1D7UikDc3xi5ATihg5kQEt', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>103</th>
      <td>30/10/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDbwfiHjkjMWsfTKNt6naeqPzOe', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>104</th>
      <td>06/11/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>134208.0</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>{'id': 'cmpl-63YDbWAndtsRqPTi2ZHZtPodZvOwr', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>105</th>
      <td>06/11/2017</td>
      <td>ALDL</td>
      <td>Legal Deposit Services</td>
      <td>27067.0</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
      <td>{'id': 'cmpl-63YDbDu7WM3svYWsRAMdDUKtSFDBu', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>106</th>
      <td>27/11/2017</td>
      <td>Maggs Bros Ltd</td>
      <td>Literary &amp; Archival Items</td>
      <td>26500.0</td>
      <td>Supplier: Maggs Bros Ltd; Description: Literar.</td>
      <td>{'id': 'cmpl-63YDbxNNI8ZH5CJJNxQ0IF9Zf925C', '.</td>
      <td>0</td>
    </tr>
    <tr>
      <th>107</th>
      <td>30/11/2017</td>
      <td>Glasgow City Council</td>
      <td>Kelvin Hall</td>
      <td>42345.0</td>
      <td>Supplier: Glasgow City Council; Description: K.


key: _node_content
string_value:
{"id_": "64729878-a8ef-4c2c-90c2-aff27fc9a8e1", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "eb1b193e-bef8-428c-8e2b-250b4add5e7f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4b7b6223a1dcc2271021d632496789fb5d39c4800025caf8159bf1ae16ae2747", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b7080157-3d41-4840-94a2-bb1c5cd44360", "node_type": "1", "metadata": {}, "hash": "abfec973cde8007ec7c6fb26613ad619f452e242d47e7e2d42b2bb3da2b0a909", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>{'id': 'cmpl-63YDbWAndtsRqPTi2ZHZtPodZvOwr', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>105</th>
      <td>06/11/2017</td>
      <td>ALDL</td>
      <td>Legal Deposit Services</td>
      <td>27067.0</td>
      <td>Supplier: ALDL; Description: Legal Deposit Ser.</td>
      <td>{'id': 'cmpl-63YDbDu7WM3svYWsRAMdDUKtSFDBu', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>106</th>
      <td>27/11/2017</td>
      <td>Maggs Bros Ltd</td>
      <td>Literary &amp; Archival Items</td>
      <td>26500.0</td>
      <td>Supplier: Maggs Bros Ltd; Description: Literar.</td>
      <td>{'id': 'cmpl-63YDbxNNI8ZH5CJJNxQ0IF9Zf925C', '.</td>
      <td>0</td>
    </tr>
    <tr>
      <th>107</th>
      <td>30/11/2017</td>
      <td>Glasgow City Council</td>
      <td>Kelvin Hall</td>
      <td>42345.0</td>
      <td>Supplier: Glasgow City Council; Description: K.</td>
      <td>{'id': 'cmpl-63YDb8R1FWu4bjwM2xE775rouwneV', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>108</th>
      <td>11/12/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDcAPsp37WhbPs9kwfUX0kBk7Hv', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>109</th>
      <td>11/12/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>159275.0</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>{'id': 'cmpl-63YDcML2welrC3wF0nuKgcNmVu1oQ', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>110</th>
      <td>08/01/2018</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.


key: _node_content
string_value:
{"id_": "b7080157-3d41-4840-94a2-bb1c5cd44360", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "fe0da3db-4d84-4600-b39a-29c7e7a81c72", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fc4c48b49d56595ea7ad607e14f7ef581cbedfabef444f56264294edcc0a2f22", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "64729878-a8ef-4c2c-90c2-aff27fc9a8e1", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Multiclass_classification_for_transactions.ipynb", "title": "Multiclass Classification for Transactions", "tokens": 12990, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b0064fb06ff961a0affeecf8f068dd617c0a8ce55dfd167e6759c9c0e74e9eac", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0</td>
      <td>Supplier: Glasgow City Council; Description: K.</td>
      <td>{'id': 'cmpl-63YDb8R1FWu4bjwM2xE775rouwneV', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>108</th>
      <td>11/12/2017</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDcAPsp37WhbPs9kwfUX0kBk7Hv', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>109</th>
      <td>11/12/2017</td>
      <td>John Graham Construction Ltd</td>
      <td>Causewayside Refurbishment</td>
      <td>159275.0</td>
      <td>Supplier: John Graham Construction Ltd; Descri.</td>
      <td>{'id': 'cmpl-63YDcML2welrC3wF0nuKgcNmVu1oQ', '.</td>
      <td>2</td>
    </tr>
    <tr>
      <th>110</th>
      <td>08/01/2018</td>
      <td>ECG Facilities Service</td>
      <td>Facilities Management Charge</td>
      <td>35758.0</td>
      <td>Supplier: ECG Facilities Service; Description:.</td>
      <td>{'id': 'cmpl-63YDc95SSdOHnIliFB2cjMEEm7Z2u', '.</td>
      <td>2</td>
    </tr>
  </tbody>
</table>
</div>




```python
holdout_df['pred'].value_counts()

```




     2    231
     0     27
    Name: pred, dtype: int64



Well those results were similarly underwhelming - so we've learned that with a dataset with a small number of labelled observations, either zero-shot classification or traditional classification with embeddings return better results than a fine-tuned model.

A fine-tuned model is still a great tool, but is more effective when you have a larger number of labelled examples for each class that you're looking to classify


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb


key: _node_content
string_value:
{"id_": "a4c7f714-6f15-4179-b15f-2c5029df642e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9eec1e1a-24d0-43b0-bc0c-4f8c5c8c7e01", "node_type": "1", "metadata": {}, "hash": "7c20e1bdbd0df5823f2245100f2a26f633fe27f77a737e064f1b358656adcafd", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3604, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to make your completions outputs reproducible with the new seed parameter

**TLDR**: Developers can now specify `seed` parameter in the Chat Completion request to receive (mostly) consistent outputs. To help you keep track of these changes, we expose the `system_fingerprint` field. If this value is different, you may see different outputs due to changes we've made on our systems. Please note that this feature is in beta and only currently supported for `gpt-4-1106-preview` and `gpt-3.5-turbo-1106`.

### Context

Reproducibility has always been a big request from user communities when using our APIs. For instance, when granted the capability of getting reproducible numerical result, users can unlock quite a bit of use cases that’s sensitive to numerical changes.

#### Model level features for consistent outputs

The Chat Completions and Completions APIs are non-deterministic by default (which means model outputs may differ from request to request), but now offer some control towards deterministic outputs using a few model level controls.

This can unlock consistent completions which enables full control on the model behaviors for anything built on top of the APIs, and quite useful for reproducing results and testing so you know get peace of mind from knowing exactly what you’d get.

#### Implementing consistent outputs

To receive _mostly_ deterministic outputs across API calls:

- Set the `seed` parameter to any integer of your choice, but use the same value across requests. For example, `12345`.
- Set all other parameters (prompt, temperature, top_p, etc.) to the same values across requests.
- In the response, check the `system_fingerprint` field. The system fingerprint is an identifier for the current combination of model weights, infrastructure, and other configuration options used by OpenAI servers to generate the completion. It changes whenever you change request parameters, or OpenAI updates numerical configuration of the infrastructure serving our models (which may happen a few times a year).

If the `seed`, request parameters, and `system_fingerprint` all match across your requests, then model outputs will mostly be identical. There is a small chance that responses differ even when request parameters and `system_fingerprint` match, due to the inherent non-determinism of our models.


### Model level controls for consistent outputs - `seed` and `system_fingerprint`

##### `seed`

If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend.

##### `system_fingerprint`

This fingerprint represents the backend configuration that the model runs with. It can be used in conjunction with the seed request parameter to understand when backend changes have been made that might impact determinism.This is the indicator on whether users should expect "almost always the same result".


## Example: Generating a short excerpt with a fixed seed

In this example, we will demonstrate how to generate a short excerpt using a fixed seed. This can be particularly useful in scenarios where you need to generate consistent results for testing, debugging, or for applications that require consistent outputs.

### Python SDK

> **Note**
> Switch to latest version of the SDK (`1.3.3` at time of writing).


```python
!pip install --upgrade openai # Switch to the latest version of OpenAI (1.3.3 at time of writing)
```


key: _node_content
string_value:
{"id_": "9eec1e1a-24d0-43b0-bc0c-4f8c5c8c7e01", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a4c7f714-6f15-4179-b15f-2c5029df642e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "970cf42730c5c3a506183c175cbff46f601fa4d941ea7508e4ed2726ee1befb5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b16ea12c-3a87-46d0-b960-e5b9330bf8b7", "node_type": "1", "metadata": {}, "hash": "452b0b508ff2e0148a182b8c6cc9fe5796a9e13ebe142bbd32ee1c056331adaa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2339, "end_char_idx": 5909, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Model level controls for consistent outputs - `seed` and `system_fingerprint`

##### `seed`

If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the `system_fingerprint` response parameter to monitor changes in the backend.

##### `system_fingerprint`

This fingerprint represents the backend configuration that the model runs with. It can be used in conjunction with the seed request parameter to understand when backend changes have been made that might impact determinism.This is the indicator on whether users should expect "almost always the same result".


## Example: Generating a short excerpt with a fixed seed

In this example, we will demonstrate how to generate a short excerpt using a fixed seed. This can be particularly useful in scenarios where you need to generate consistent results for testing, debugging, or for applications that require consistent outputs.

### Python SDK

> **Note**
> Switch to latest version of the SDK (`1.3.3` at time of writing).


```python
!pip install --upgrade openai # Switch to the latest version of OpenAI (1.3.3 at time of writing)
```


```python
import openai
import asyncio
from IPython.display import display, HTML

from utils.embeddings_utils import (
    get_embedding,
    distances_from_embeddings
)

GPT_MODEL = "gpt-3.5-turbo-1106"
```


```python
async def get_chat_response(
    system_message: str, user_request: str, seed: int = None, temperature: float = 0.7
):
    try:
        messages = [
            {"role": "system", "content": system_message},
            {"role": "user", "content": user_request},
        ]

        response = openai.chat.completions.create(
            model=GPT_MODEL,
            messages=messages,
            seed=seed,
            max_tokens=200,
            temperature=temperature,
        )

        response_content = response.choices[0].message.content
        system_fingerprint = response.system_fingerprint
        prompt_tokens = response.usage.prompt_tokens
        completion_tokens = response.usage.total_tokens - response.usage.prompt_tokens

        table = f"""
        <table>
        <tr><th>Response</th><td>{response_content}</td></tr>
        <tr><th>System Fingerprint</th><td>{system_fingerprint}</td></tr>
        <tr><th>Number of prompt tokens</th><td>{prompt_tokens}</td></tr>
        <tr><th>Number of completion tokens</th><td>{completion_tokens}</td></tr>
        </table>
        """
        display(HTML(table))

        return response_content
    except Exception as e:
        print(f"An error occurred: {e}")
        return None

def calculate_average_distance(responses):
    """
    This function calculates the average distance between the embeddings of the responses.
    The distance between embeddings is a measure of how similar the responses are.
    """
    # Calculate embeddings for each response
    response_embeddings = [get_embedding(response) for response in responses]

    # Compute distances between the first response and the rest
    distances = distances_from_embeddings(response_embeddings[0], response_embeddings[1:])

    # Calculate the average distance
    average_distance = sum(distances) / len(distances)

    # Return the average distance
    return average_distance
```

First, let's try generating few different versions of a short excerpt about "a journey to Mars" without the `seed` parameter. This is the default behavior:


key: _node_content
string_value:
{"id_": "b16ea12c-3a87-46d0-b960-e5b9330bf8b7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9eec1e1a-24d0-43b0-bc0c-4f8c5c8c7e01", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "111dde55645e3d287763abdefb263cbb4a9d705f79e9958a32fd3e5f7ef6e1cc", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ef3b0d2d-cd44-41bb-a643-134f597be0eb", "node_type": "1", "metadata": {}, "hash": "cf628a3cab1fa679802d2f8ab48a76df5b7bf78401beb764129ac7376cd4a756", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5912, "end_char_idx": 8535, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
topic = "a journey to Mars"
system_message = "You are a helpful assistant."
user_request = f"Generate a short excerpt of news about {topic}."

responses = []


async def get_response(i):
    print(f'Output {i + 1}\n{"-" * 10}')
    response = await get_chat_response(
        system_message=system_message, user_request=user_request
    )
    return response


responses = await asyncio.gather(*[get_response(i) for i in range(5)])
average_distance = calculate_average_distance(responses)
print(f"The average similarity between responses is: {average_distance}")
```

    Output 1
    ----------




<table>
<tr><th>Response</th><td>"NASA's Mars mission reaches critical stage as spacecraft successfully enters orbit around the red planet. The historic journey, which began over a year ago, has captured the world's attention as scientists and astronauts prepare to land on Mars for the first time. The mission is expected to provide valuable insights into the planet's geology, atmosphere, and potential for sustaining human life in the future."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>76</td></tr>
</table>



    Output 2
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully landed on Mars, marking a major milestone in the mission to explore the red planet. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples of rock and soil for future return to Earth. This historic achievement paves the way for further exploration and potential human missions to Mars in the near future."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>76</td></tr>
</table>



    Output 3
    ----------




<table>
<tr><th>Response</th><td>"SpaceX successfully launched the first manned mission to Mars yesterday, marking a historic milestone in space exploration. The crew of four astronauts will spend the next six months traveling to the red planet, where they will conduct groundbreaking research and experiments. This mission represents a significant step towards establishing a human presence on Mars and paves the way for future interplanetary travel."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 4
    ----------


key: _node_content
string_value:
{"id_": "ef3b0d2d-cd44-41bb-a643-134f597be0eb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b16ea12c-3a87-46d0-b960-e5b9330bf8b7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "7d1ec7b8525c5906da22f5af5878479dd80b664f5b54c3d728f4c035dca1df29", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4e9d9fd8-c7aa-4e7a-bd49-18bf852b3e43", "node_type": "1", "metadata": {}, "hash": "6a650747ec41da3ca78d938b744590209f4b6439e8672e77c8e63d47670f6053", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7164, "end_char_idx": 9157, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Output 2
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully landed on Mars, marking a major milestone in the mission to explore the red planet. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples of rock and soil for future return to Earth. This historic achievement paves the way for further exploration and potential human missions to Mars in the near future."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>76</td></tr>
</table>



    Output 3
    ----------




<table>
<tr><th>Response</th><td>"SpaceX successfully launched the first manned mission to Mars yesterday, marking a historic milestone in space exploration. The crew of four astronauts will spend the next six months traveling to the red planet, where they will conduct groundbreaking research and experiments. This mission represents a significant step towards establishing a human presence on Mars and paves the way for future interplanetary travel."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 4
    ----------




<table>
<tr><th>Response</th><td>"NASA's latest Mars mission exceeds expectations as the Perseverance rover uncovers tantalizing clues about the Red Planet's past. Scientists are thrilled by the discovery of ancient riverbeds and sedimentary rocks, raising hopes of finding signs of past life on Mars. With this exciting progress, the dream of sending humans to Mars feels closer than ever before."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 5
    ----------


key: _node_content
string_value:
{"id_": "4e9d9fd8-c7aa-4e7a-bd49-18bf852b3e43", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ef3b0d2d-cd44-41bb-a643-134f597be0eb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "429b8de10b6951a920874154e2b6aaecc1dca11a4c6792014f665ee617e4997f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c1f3d455-7439-4742-9daf-126e2e1f67ab", "node_type": "1", "metadata": {}, "hash": "be9f633f079e4029782524c3ca27d1563aac1cfeace59371a5c018e5916669ef", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7836, "end_char_idx": 10972, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Output 3
    ----------




<table>
<tr><th>Response</th><td>"SpaceX successfully launched the first manned mission to Mars yesterday, marking a historic milestone in space exploration. The crew of four astronauts will spend the next six months traveling to the red planet, where they will conduct groundbreaking research and experiments. This mission represents a significant step towards establishing a human presence on Mars and paves the way for future interplanetary travel."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 4
    ----------




<table>
<tr><th>Response</th><td>"NASA's latest Mars mission exceeds expectations as the Perseverance rover uncovers tantalizing clues about the Red Planet's past. Scientists are thrilled by the discovery of ancient riverbeds and sedimentary rocks, raising hopes of finding signs of past life on Mars. With this exciting progress, the dream of sending humans to Mars feels closer than ever before."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 5
    ----------




        <table>
        <tr><th>Response</th><td>"NASA's Perseverance Rover Successfully Lands on Mars, Begins Exploration Mission

In a historic moment for space exploration, NASA's Perseverance rover has successfully landed on the surface of Mars. After a seven-month journey, the rover touched down in the Jezero Crater, a location scientists believe may have once held a lake and could potentially contain signs of ancient microbial life.

The rover's primary mission is to search for evidence of past life on Mars and collect rock and soil samples for future return to Earth. Equipped with advanced scientific instruments, including cameras, spectrometers, and a drill, Perseverance will begin its exploration of the Martian surface, providing valuable data and insights into the planet's geology and potential habitability.

This successful landing marks a significant milestone in humanity's quest to understand the red planet and paves the way for future manned missions to Mars. NASA's Perseverance rover is poised to unravel the mysteries of Mars and unlock new possibilities</td></tr>
        <tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
        <tr><th>Number of prompt tokens</th><td>29</td></tr>
        <tr><th>Number of completion tokens</th><td>200</td></tr>
        </table>



    The average similarity between responses is: 0.1136714512418833


Now, let's try to tun the same code with a constant `seed` of 123 and `temperature` of 0 and compare the responses and `system_fingerprint`.


```python
SEED = 123
responses = []


async def get_response(i):
    print(f'Output {i + 1}\n{"-" * 10}')
    response = await get_chat_response(
        system_message=system_message,
        seed=SEED,
        temperature=0,
        user_request=user_request,
    )
    return response


key: _node_content
string_value:
{"id_": "c1f3d455-7439-4742-9daf-126e2e1f67ab", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4e9d9fd8-c7aa-4e7a-bd49-18bf852b3e43", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "5f7e1bf45304cb19142f8167272f39d7f20d81b772aaf01f0604b38b12142a5b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fe0769ca-5617-4428-af07-ea94f85dd7f7", "node_type": "1", "metadata": {}, "hash": "8a4ac5e19a41803da97e1309d1421764bd32c3b4424fd3b29dba391de9761c35", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10476, "end_char_idx": 13474, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The average similarity between responses is: 0.1136714512418833


Now, let's try to tun the same code with a constant `seed` of 123 and `temperature` of 0 and compare the responses and `system_fingerprint`.


```python
SEED = 123
responses = []


async def get_response(i):
    print(f'Output {i + 1}\n{"-" * 10}')
    response = await get_chat_response(
        system_message=system_message,
        seed=SEED,
        temperature=0,
        user_request=user_request,
    )
    return response


responses = await asyncio.gather(*[get_response(i) for i in range(5)])

average_distance = calculate_average_distance(responses)
print(f"The average distance between responses is: {average_distance}")
```

    Output 1
    ----------




        <table>
        <tr><th>Response</th><td>"NASA's Perseverance Rover Successfully Lands on Mars

In a historic achievement, NASA's Perseverance rover has successfully landed on the surface of Mars, marking a major milestone in the exploration of the red planet. The rover, which traveled over 293 million miles from Earth, is equipped with state-of-the-art instruments designed to search for signs of ancient microbial life and collect rock and soil samples for future return to Earth. This mission represents a significant step forward in our understanding of Mars and the potential for human exploration of the planet in the future."</td></tr>
        <tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
        <tr><th>Number of prompt tokens</th><td>29</td></tr>
        <tr><th>Number of completion tokens</th><td>113</td></tr>
        </table>



    Output 2
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel and expand our understanding of the universe."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>81</td></tr>
</table>



    Output 3
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as NASA continues to push the boundaries of space exploration."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 4
    ----------


key: _node_content
string_value:
{"id_": "fe0769ca-5617-4428-af07-ea94f85dd7f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c1f3d455-7439-4742-9daf-126e2e1f67ab", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "90795511d3c57f7513ee597d9c32c3e97339ed6c3524e96c803f6f3d58c838c9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "87db346b-9aa4-4750-9f0e-d44181218716", "node_type": "1", "metadata": {}, "hash": "f6d42f6cbf175eaa37b6ab4806fa3a329ace09ea884bd8515fd1444b5b8abbda", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12084, "end_char_idx": 14893, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Output 2
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel and expand our understanding of the universe."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>81</td></tr>
</table>



    Output 3
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as NASA continues to push the boundaries of space exploration."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>72</td></tr>
</table>



    Output 4
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel and expand our understanding of the universe."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>81</td></tr>
</table>



    Output 5
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>74</td></tr>
</table>



    The average distance between responses is: 0.0449054397632461


key: _node_content
string_value:
{"id_": "87db346b-9aa4-4750-9f0e-d44181218716", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3aa675d5-e5bf-45f7-b282-c0e00ef66c24", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "341fc86701feda05a05ae6e7e366b1a273d28b8a0661de95ef5d1b4861964df2", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fe0769ca-5617-4428-af07-ea94f85dd7f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Reproducible_outputs_with_the_seed_parameter.ipynb", "title": "How to make your completions outputs reproducible with the new seed parameter", "tokens": 3546, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3fee596ecb86eda3abf729adeaad7bd2c38615c64b4e2ffa4eb57f01046a4af8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13451, "end_char_idx": 15788, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Output 4
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel and expand our understanding of the universe."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>81</td></tr>
</table>



    Output 5
    ----------




<table>
<tr><th>Response</th><td>"NASA's Perseverance rover successfully lands on Mars, marking a historic milestone in space exploration. The rover is equipped with advanced scientific instruments to search for signs of ancient microbial life and collect samples for future return to Earth. This mission paves the way for future human exploration of the red planet, as scientists and engineers continue to push the boundaries of space travel."</td></tr>
<tr><th>System Fingerprint</th><td>fp_772e8125bb</td></tr>
<tr><th>Number of prompt tokens</th><td>29</td></tr>
<tr><th>Number of completion tokens</th><td>74</td></tr>
</table>



    The average distance between responses is: 0.0449054397632461


As we can observe, the `seed` parameter allows us to generate much more consistent results.

## Conclusion

We demonstrated how to use a fixed integer `seed` to generate consistent outputs from our model. This is particularly useful in scenarios where reproducibility is important. However, it's important to note that while the `seed` ensures consistency, it does not guarantee the quality of the output. Note that when you want to use reproducible outputs, you need to set the `seed` to the same integer across Chat Completions calls. You should also match any other parameters like `temperature`, `max_tokens` etc. Further extension of reproducible outputs could be to use consistent `seed` when benchmarking/evaluating the performance of different prompts or models, to ensure that each version is evaluated under the same conditions, making the comparisons fair and the results reliable.


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb


key: _node_content
string_value:
{"id_": "ea5b6f0f-2007-4b6f-add7-857da087015a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bc2e658a-dba9-4e94-ac87-c7d87a225110", "node_type": "1", "metadata": {}, "hash": "ec8ce607d441be283d30b2a7b574ffca9393602129182a47a8229fb738013bbe", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2383, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Question answering using embeddings-based search

GPT excels at answering questions, but only on topics it remembers from its training data.

What should you do if you want GPT to answer questions about unfamiliar topics? E.g.,
- Recent events after Sep 2021
- Your non-public documents
- Information from past conversations
- etc.

This notebook demonstrates a two-step Search-Ask method for enabling GPT to answer questions using a library of reference text.

1. **Search:** search your library of text for relevant text sections
2. **Ask:** insert the retrieved text sections into a message to GPT and ask it the question

## Why search is better than fine-tuning

GPT can learn knowledge in two ways:

- Via model weights (i.e., fine-tune the model on a training set)
- Via model inputs (i.e., insert the knowledge into an input message)

Although fine-tuning can feel like the more natural option—training on data is how GPT learned all of its other knowledge, after all—we generally do not recommend it as a way to teach the model knowledge. Fine-tuning is better suited to teaching specialized tasks or styles, and is less reliable for factual recall.

As an analogy, model weights are like long-term memory. When you fine-tune a model, it's like studying for an exam a week away. When the exam arrives, the model may forget details, or misremember facts it never read.

In contrast, message inputs are like short-term memory. When you insert knowledge into a message, it's like taking an exam with open notes. With notes in hand, the model is more likely to arrive at correct answers.

One downside of text search relative to fine-tuning is that each model is limited by a maximum amount of text it can read at once:

| Model           | Maximum text length       |
|-----------------|---------------------------|
| `gpt-3.5-turbo` | 4,096 tokens (~5 pages)   |
| `gpt-4`         | 8,192 tokens (~10 pages)  |
| `gpt-4-32k`     | 32,768 tokens (~40 pages) |

(New model is available with longer contexts, gpt-4-1106-preview have 128K context window)

Continuing the analogy, you can think of the model like a student who can only look at a few pages of notes at a time, despite potentially having shelves of textbooks to draw upon.

Therefore, to build a system capable of drawing upon large quantities of text to answer questions, we recommend using a Search-Ask approach.


key: _node_content
string_value:
{"id_": "bc2e658a-dba9-4e94-ac87-c7d87a225110", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ea5b6f0f-2007-4b6f-add7-857da087015a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "bdf8a55a013e3648cf0e4d3eae3d15210e94f52d6efd26dba76762d9b1ac5754", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "428d8ef5-8684-422d-be8b-f18a8afcd159", "node_type": "1", "metadata": {}, "hash": "857a0397b421cdbe0885c0e1aa064194aefecb402ade1cd32815837e1f03ad65", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2386, "end_char_idx": 4724, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Search

Text can be searched in many ways. E.g.,

- Lexical-based search
- Graph-based search
- Embedding-based search

This example notebook uses embedding-based search. [Embeddings](https://platform.openai.com/docs/guides/embeddings) are simple to implement and work especially well with questions, as questions often don't lexically overlap with their answers.

Consider embeddings-only search as a starting point for your own system. Better search systems might combine multiple search methods, along with features like popularity, recency, user history, redundancy with prior search results, click rate data, etc. Q&A retrieval performance may also be improved with techniques like [HyDE](https://arxiv.org/abs/2212.10496), in which questions are first transformed into hypothetical answers before being embedded. Similarly, GPT can also potentially improve search results by automatically transforming questions into sets of keywords or search terms.

## Full procedure

Specifically, this notebook demonstrates the following procedure:

1. Prepare search data (once per document)
    1. Collect: We'll download a few hundred Wikipedia articles about the 2022 Olympics
    2. Chunk: Documents are split into short, mostly self-contained sections to be embedded
    3. Embed: Each section is embedded with the OpenAI API
    4. Store: Embeddings are saved (for large datasets, use a vector database)
2. Search (once per query)
    1. Given a user question, generate an embedding for the query from the OpenAI API
    2. Using the embeddings, rank the text sections by relevance to the query
3. Ask (once per query)
    1. Insert the question and the most relevant sections into a message to GPT
    2. Return GPT's answer

### Costs

Because GPT is more expensive than embeddings search, a system with a decent volume of queries will have its costs dominated by step 3.

- For `gpt-3.5-turbo` using ~1,000 tokens per query, it costs ~$0.002 per query, or ~500 queries per dollar (as of Apr 2023)
- For `gpt-4`, again assuming ~1,000 tokens per query, it costs ~$0.03 per query, or ~30 queries per dollar (as of Apr 2023)

Of course, exact costs will depend on the system specifics and usage patterns.

## Preamble

We'll begin by:
- Importing the necessary libraries
- Selecting models for embeddings search and question answering


key: _node_content
string_value:
{"id_": "428d8ef5-8684-422d-be8b-f18a8afcd159", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "bc2e658a-dba9-4e94-ac87-c7d87a225110", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "da9d238188d887b78754dadc6300722e25fa040b37d55aa9a1855a762e14e919", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4d5715dd-d434-40e8-b6d2-441beab3d1ce", "node_type": "1", "metadata": {}, "hash": "4bee4a281c37bfe174b3c7680e373adb8c826f0718b170caf1a6752b6596041a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4729, "end_char_idx": 7616, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# imports
import ast  # for converting embeddings saved as strings back to arrays
from openai import OpenAI # for calling the OpenAI API
import pandas as pd  # for storing text and embeddings data
import tiktoken  # for counting tokens
import os # for getting API token from env variable OPENAI_API_KEY
from scipy import spatial  # for calculating vector similarities for search

# models
EMBEDDING_MODEL = "text-embedding-ada-002"
GPT_MODEL = "gpt-3.5-turbo"

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

```

#### Troubleshooting: Installing libraries

If you need to install any of the libraries above, run `pip install {library_name}` in your terminal.

For example, to install the `openai` library, run:
```zsh
pip install openai
```

(You can also do this in a notebook cell with `!pip install openai` or `%pip install openai`.)

After installing, restart the notebook kernel so the libraries can be loaded.

#### Troubleshooting: Setting your API key

The OpenAI library will try to read your API key from the `OPENAI_API_KEY` environment variable. If you haven't already, you can set this environment variable by following [these instructions](https://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety).

### Motivating example: GPT cannot answer questions about current events

Because the training data for `gpt-3.5-turbo` and `gpt-4` mostly ends in September 2021, the models cannot answer questions about more recent events, such as the 2022 Winter Olympics.

For example, let's try asking 'Which athletes won the gold medal in curling in 2022?':


```python
# an example question about the 2022 Olympics
query = 'Which athletes won the gold medal in curling at the 2022 Winter Olympics?'

response = client.chat.completions.create(
    messages=[
        {'role': 'system', 'content': 'You answer questions about the 2022 Winter Olympics.'},
        {'role': 'user', 'content': query},
    ],
    model=GPT_MODEL,
    temperature=0,
)

print(response.choices[0].message.content)
```

    As an AI language model, I don't have real-time data. However, I can provide you with general information. The gold medalists in curling at the 2022 Winter Olympics will be determined during the event. The winners will be the team that finishes in first place in the respective men's and women's curling competitions. To find out the specific gold medalists, you can check the official Olympic website or reliable news sources for the most up-to-date information.


In this case, the model has no knowledge of 2022 and is unable to answer the question.

### You can give GPT knowledge about a topic by inserting it into an input message

To help give the model knowledge of curling at the 2022 Winter Olympics, we can copy and paste the top half of a relevant Wikipedia article into our message:


key: _node_content
string_value:
{"id_": "4d5715dd-d434-40e8-b6d2-441beab3d1ce", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "428d8ef5-8684-422d-be8b-f18a8afcd159", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "d2d33218722954874e8a9670fc537e28082f7199098c84593b96538b939bac89", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "07a54773-5763-4a30-9d23-438565916ba8", "node_type": "1", "metadata": {}, "hash": "3c6d5fdb6a4afa681162fe02bde0d43024b941b1ee3f0ba564798e1afab16b87", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6381, "end_char_idx": 9329, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# an example question about the 2022 Olympics
query = 'Which athletes won the gold medal in curling at the 2022 Winter Olympics?'

response = client.chat.completions.create(
    messages=[
        {'role': 'system', 'content': 'You answer questions about the 2022 Winter Olympics.'},
        {'role': 'user', 'content': query},
    ],
    model=GPT_MODEL,
    temperature=0,
)

print(response.choices[0].message.content)
```

    As an AI language model, I don't have real-time data. However, I can provide you with general information. The gold medalists in curling at the 2022 Winter Olympics will be determined during the event. The winners will be the team that finishes in first place in the respective men's and women's curling competitions. To find out the specific gold medalists, you can check the official Olympic website or reliable news sources for the most up-to-date information.


In this case, the model has no knowledge of 2022 and is unable to answer the question.

### You can give GPT knowledge about a topic by inserting it into an input message

To help give the model knowledge of curling at the 2022 Winter Olympics, we can copy and paste the top half of a relevant Wikipedia article into our message:


```python
# text copied and pasted from: https://en.wikipedia.org/wiki/Curling_at_the_2022_Winter_Olympics
# I didn't bother to format or clean the text, but GPT will still understand it
# the entire article is too long for gpt-3.5-turbo, so I only included the top few sections

wikipedia_article_on_curling = """Curling at the 2022 Winter Olympics

Article
Talk
Read
Edit
View history
From Wikipedia, the free encyclopedia
Curling
at the XXIV Olympic Winter Games
Curling pictogram.svg
Curling pictogram
Venue	Beijing National Aquatics Centre
Dates	2–20 February 2022
No. of events	3 (1 men, 1 women, 1 mixed)
Competitors	114 from 14 nations
← 20182026 →
Men's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Sweden
2nd place, silver medalist(s)		 Great Britain
3rd place, bronze medalist(s)		 Canada
Women's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Great Britain
2nd place, silver medalist(s)		 Japan
3rd place, bronze medalist(s)		 Sweden
Mixed doubles's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Italy
2nd place, silver medalist(s)		 Norway
3rd place, bronze medalist(s)		 Sweden
Curling at the
2022 Winter Olympics
Curling pictogram.svg
Qualification
Statistics
Tournament
Men
Women
Mixed doubles
vte
The curling competitions of the 2022 Winter Olympics were held at the Beijing National Aquatics Centre, one of the Olympic Green venues. Curling competitions were scheduled for every day of the games, from February 2 to February 20.[1] This was the eighth time that curling was part of the Olympic program.

In each of the men's, women's, and mixed doubles competitions, 10 nations competed.


key: _node_content
string_value:
{"id_": "07a54773-5763-4a30-9d23-438565916ba8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4d5715dd-d434-40e8-b6d2-441beab3d1ce", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2b1e3f2070045acf6cfba6ba2c3b30dd31be35dd833e04328b8c8da076e39013", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "96567f9b-a669-40bb-ac83-d3db0689236f", "node_type": "1", "metadata": {}, "hash": "a1cedd0df48286be7850c0bf677e6ef2589dc594dd900add1bdebb6a2fadf502", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8193, "end_char_idx": 11070, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
of events	3 (1 men, 1 women, 1 mixed)
Competitors	114 from 14 nations
← 20182026 →
Men's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Sweden
2nd place, silver medalist(s)		 Great Britain
3rd place, bronze medalist(s)		 Canada
Women's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Great Britain
2nd place, silver medalist(s)		 Japan
3rd place, bronze medalist(s)		 Sweden
Mixed doubles's curling
at the XXIV Olympic Winter Games
Medalists
1st place, gold medalist(s)		 Italy
2nd place, silver medalist(s)		 Norway
3rd place, bronze medalist(s)		 Sweden
Curling at the
2022 Winter Olympics
Curling pictogram.svg
Qualification
Statistics
Tournament
Men
Women
Mixed doubles
vte
The curling competitions of the 2022 Winter Olympics were held at the Beijing National Aquatics Centre, one of the Olympic Green venues. Curling competitions were scheduled for every day of the games, from February 2 to February 20.[1] This was the eighth time that curling was part of the Olympic program.

In each of the men's, women's, and mixed doubles competitions, 10 nations competed. The mixed doubles competition was expanded for its second appearance in the Olympics.[2] A total of 120 quota spots (60 per sex) were distributed to the sport of curling, an increase of four from the 2018 Winter Olympics.[3] A total of 3 events were contested, one for men, one for women, and one mixed.[4]

Qualification
Main article: Curling at the 2022 Winter Olympics – Qualification
Qualification to the Men's and Women's curling tournaments at the Winter Olympics was determined through two methods (in addition to the host nation). Nations qualified teams by placing in the top six at the 2021 World Curling Championships. Teams could also qualify through Olympic qualification events which were held in 2021. Six nations qualified via World Championship qualification placement, while three nations qualified through qualification events. In men's and women's play, a host will be selected for the Olympic Qualification Event (OQE). They would be joined by the teams which competed at the 2021 World Championships but did not qualify for the Olympics, and two qualifiers from the Pre-Olympic Qualification Event (Pre-OQE). The Pre-OQE was open to all member associations.[5]

For the mixed doubles competition in 2022, the tournament field was expanded from eight competitor nations to ten.[2] The top seven ranked teams at the 2021 World Mixed Doubles Curling Championship qualified, along with two teams from the Olympic Qualification Event (OQE) – Mixed Doubles. This OQE was open to a nominated host and the fifteen nations with the highest qualification points not already qualified to the Olympics. As the host nation, China qualified teams automatically, thus making a total of ten teams per event in the curling tournaments.


key: _node_content
string_value:
{"id_": "96567f9b-a669-40bb-ac83-d3db0689236f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "07a54773-5763-4a30-9d23-438565916ba8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f935d62d993123f7fc9c7f344229c3d672f632746416c57a0d839e88e5de2fbf", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9c9987b9-1c94-48d9-8b97-d2300f0e7c21", "node_type": "1", "metadata": {}, "hash": "381883dc585e53bab3c71d8ae3c3ead40497d1973d1085f5ef432514fae3829e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9246, "end_char_idx": 11763, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
In each of the men's, women's, and mixed doubles competitions, 10 nations competed. The mixed doubles competition was expanded for its second appearance in the Olympics.[2] A total of 120 quota spots (60 per sex) were distributed to the sport of curling, an increase of four from the 2018 Winter Olympics.[3] A total of 3 events were contested, one for men, one for women, and one mixed.[4]

Qualification
Main article: Curling at the 2022 Winter Olympics – Qualification
Qualification to the Men's and Women's curling tournaments at the Winter Olympics was determined through two methods (in addition to the host nation). Nations qualified teams by placing in the top six at the 2021 World Curling Championships. Teams could also qualify through Olympic qualification events which were held in 2021. Six nations qualified via World Championship qualification placement, while three nations qualified through qualification events. In men's and women's play, a host will be selected for the Olympic Qualification Event (OQE). They would be joined by the teams which competed at the 2021 World Championships but did not qualify for the Olympics, and two qualifiers from the Pre-Olympic Qualification Event (Pre-OQE). The Pre-OQE was open to all member associations.[5]

For the mixed doubles competition in 2022, the tournament field was expanded from eight competitor nations to ten.[2] The top seven ranked teams at the 2021 World Mixed Doubles Curling Championship qualified, along with two teams from the Olympic Qualification Event (OQE) – Mixed Doubles. This OQE was open to a nominated host and the fifteen nations with the highest qualification points not already qualified to the Olympics. As the host nation, China qualified teams automatically, thus making a total of ten teams per event in the curling tournaments.[6]

Summary
Nations	Men	Women	Mixed doubles	Athletes
 Australia			Yes	2
 Canada	Yes	Yes	Yes	12
 China	Yes	Yes	Yes	12
 Czech Republic			Yes	2
 Denmark	Yes	Yes		10
 Great Britain	Yes	Yes	Yes	10
 Italy	Yes		Yes	6
 Japan		Yes		5
 Norway	Yes		Yes	6
 ROC	Yes	Yes		10
 South Korea		Yes		5
 Sweden	Yes	Yes	Yes	11
 Switzerland	Yes	Yes	Yes	12
 United States	Yes	Yes	Yes	11
Total: 14 NOCs	10	10	10	114
Competition schedule

The Beijing National Aquatics Centre served as the venue of the curling competitions.
Curling competitions started two days before the Opening Ceremony and finished on the last day of the games, meaning the sport was the only one to have had a competition every day of the games.


key: _node_content
string_value:
{"id_": "9c9987b9-1c94-48d9-8b97-d2300f0e7c21", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "96567f9b-a669-40bb-ac83-d3db0689236f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0f5f12c1865ea0ecf9df3a9b70e107ec07d26e2fbda92bf725c9789c4b556d63", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a4fbcbd0-7c8c-46a4-be91-6b5d8cbc1fb9", "node_type": "1", "metadata": {}, "hash": "7c5483a70ab72746226f745352333e7e7deb5689f1105f7dc1625ef79a5a760d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10271, "end_char_idx": 12605, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
They would be joined by the teams which competed at the 2021 World Championships but did not qualify for the Olympics, and two qualifiers from the Pre-Olympic Qualification Event (Pre-OQE). The Pre-OQE was open to all member associations.[5]

For the mixed doubles competition in 2022, the tournament field was expanded from eight competitor nations to ten.[2] The top seven ranked teams at the 2021 World Mixed Doubles Curling Championship qualified, along with two teams from the Olympic Qualification Event (OQE) – Mixed Doubles. This OQE was open to a nominated host and the fifteen nations with the highest qualification points not already qualified to the Olympics. As the host nation, China qualified teams automatically, thus making a total of ten teams per event in the curling tournaments.[6]

Summary
Nations	Men	Women	Mixed doubles	Athletes
 Australia			Yes	2
 Canada	Yes	Yes	Yes	12
 China	Yes	Yes	Yes	12
 Czech Republic			Yes	2
 Denmark	Yes	Yes		10
 Great Britain	Yes	Yes	Yes	10
 Italy	Yes		Yes	6
 Japan		Yes		5
 Norway	Yes		Yes	6
 ROC	Yes	Yes		10
 South Korea		Yes		5
 Sweden	Yes	Yes	Yes	11
 Switzerland	Yes	Yes	Yes	12
 United States	Yes	Yes	Yes	11
Total: 14 NOCs	10	10	10	114
Competition schedule

The Beijing National Aquatics Centre served as the venue of the curling competitions.
Curling competitions started two days before the Opening Ceremony and finished on the last day of the games, meaning the sport was the only one to have had a competition every day of the games. The following was the competition schedule for the curling competitions:

RR	Round robin	SF	Semifinals	B	3rd place play-off	F	Final
Date
Event
Wed 2	Thu 3	Fri 4	Sat 5	Sun 6	Mon 7	Tue 8	Wed 9	Thu 10	Fri 11	Sat 12	Sun 13	Mon 14	Tue 15	Wed 16	Thu 17	Fri 18	Sat 19	Sun 20
Men's tournament								RR	RR	RR	RR	RR	RR	RR	RR	RR	SF	B	F	
Women's tournament									RR	RR	RR	RR	RR	RR	RR	RR	SF	B	F
Mixed doubles	RR	RR	RR	RR	RR	RR	SF	B	F												
Medal summary
Medal table
Rank	Nation	Gold	Silver	Bronze	Total
1	 Great Britain	1	1	0	2
2	 Sweden	1	0	2	3
3	 Italy	1	0	0	1
4	 Japan	0	1	0	1
 Norway	0	1	0	1
6	 Canada	0	0	1	1
Totals (6 entries)	3	3	3	9
Medalists
Event	Gold	Silver	Bronze
Men
details	 Sweden
Niklas Edin
Oskar Eriksson
Rasmus Wranå
Christoffer Sundgren
Daniel Magnusson	 Great Britain
Bruce Mouat
Grant Hardie
Bobby Lammie
Hammy McMillan Jr.


key: _node_content
string_value:
{"id_": "a4fbcbd0-7c8c-46a4-be91-6b5d8cbc1fb9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9c9987b9-1c94-48d9-8b97-d2300f0e7c21", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "88620019c3f1bdae79a327fc5d2328c69c398bb3e75e04e537b0d5e080d41ae0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "047c47b0-7e16-43fa-a730-aa844eff6620", "node_type": "1", "metadata": {}, "hash": "ec1aa3823065898965e7593abcaf3ba0b4906eb017849fe4623fcc4af850f53b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11764, "end_char_idx": 13862, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The following was the competition schedule for the curling competitions:

RR	Round robin	SF	Semifinals	B	3rd place play-off	F	Final
Date
Event
Wed 2	Thu 3	Fri 4	Sat 5	Sun 6	Mon 7	Tue 8	Wed 9	Thu 10	Fri 11	Sat 12	Sun 13	Mon 14	Tue 15	Wed 16	Thu 17	Fri 18	Sat 19	Sun 20
Men's tournament								RR	RR	RR	RR	RR	RR	RR	RR	RR	SF	B	F	
Women's tournament									RR	RR	RR	RR	RR	RR	RR	RR	SF	B	F
Mixed doubles	RR	RR	RR	RR	RR	RR	SF	B	F												
Medal summary
Medal table
Rank	Nation	Gold	Silver	Bronze	Total
1	 Great Britain	1	1	0	2
2	 Sweden	1	0	2	3
3	 Italy	1	0	0	1
4	 Japan	0	1	0	1
 Norway	0	1	0	1
6	 Canada	0	0	1	1
Totals (6 entries)	3	3	3	9
Medalists
Event	Gold	Silver	Bronze
Men
details	 Sweden
Niklas Edin
Oskar Eriksson
Rasmus Wranå
Christoffer Sundgren
Daniel Magnusson	 Great Britain
Bruce Mouat
Grant Hardie
Bobby Lammie
Hammy McMillan Jr.
Ross Whyte	 Canada
Brad Gushue
Mark Nichols
Brett Gallant
Geoff Walker
Marc Kennedy
Women
details	 Great Britain
Eve Muirhead
Vicky Wright
Jennifer Dodds
Hailey Duff
Mili Smith	 Japan
Satsuki Fujisawa
Chinami Yoshida
Yumi Suzuki
Yurika Yoshida
Kotomi Ishizaki	 Sweden
Anna Hasselborg
Sara McManus
Agnes Knochenhauer
Sofia Mabergs
Johanna Heldin
Mixed doubles
details	 Italy
Stefania Constantini
Amos Mosaner	 Norway
Kristin Skaslien
Magnus Nedregotten	 Sweden
Almida de Val
Oskar Eriksson
Teams
Men
 Canada	 China	 Denmark	 Great Britain	 Italy
Skip: Brad Gushue
Third: Mark Nichols
Second: Brett Gallant
Lead: Geoff Walker
Alternate: Marc Kennedy

Skip: Ma Xiuyue
Third: Zou Qiang
Second: Wang Zhiyu
Lead: Xu Jingtao
Alternate: Jiang Dongxu

Skip: Mikkel Krause
Third: Mads Nørgård
Second: Henrik Holtermann
Lead: Kasper Wiksten
Alternate: Tobias Thune

Skip: Bruce Mouat
Third: Grant Hardie
Second: Bobby Lammie
Lead: Hammy McMillan Jr.
Alternate: Ross Whyte

Skip: Joël Retornaz
Third: Amos Mosaner
Second: Sebastiano Arman
Lead: Simone Gonin
Alternate: Mattia Giovanella

 Norway	 ROC	 Sweden	 Switzerland	 United States
Skip: Steffen Walstad
Third: Torger Nergård
Second: Markus Høiberg
Lead: Magnus Vågberg
Alternate: Magnus Nedregotten

Skip: Sergey


key: _node_content
string_value:
{"id_": "047c47b0-7e16-43fa-a730-aa844eff6620", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a4fbcbd0-7c8c-46a4-be91-6b5d8cbc1fb9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "bd66df92e75399bd8ae1e817142a7080071f026f92186d21816ef3dd8576a624", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5def565f-806e-4ec2-8a02-04b67a31256a", "node_type": "1", "metadata": {}, "hash": "ace92ba4c80871e72d08618a9da95ce4b6176076b0790cbafc94dbb7918408fa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13545, "end_char_idx": 16036, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Alternate: Ross Whyte

Skip: Joël Retornaz
Third: Amos Mosaner
Second: Sebastiano Arman
Lead: Simone Gonin
Alternate: Mattia Giovanella

 Norway	 ROC	 Sweden	 Switzerland	 United States
Skip: Steffen Walstad
Third: Torger Nergård
Second: Markus Høiberg
Lead: Magnus Vågberg
Alternate: Magnus Nedregotten

Skip: Sergey Glukhov
Third: Evgeny Klimov
Second: Dmitry Mironov
Lead: Anton Kalalb
Alternate: Daniil Goriachev

Skip: Niklas Edin
Third: Oskar Eriksson
Second: Rasmus Wranå
Lead: Christoffer Sundgren
Alternate: Daniel Magnusson

Fourth: Benoît Schwarz
Third: Sven Michel
Skip: Peter de Cruz
Lead: Valentin Tanner
Alternate: Pablo Lachat

Skip: John Shuster
Third: Chris Plys
Second: Matt Hamilton
Lead: John Landsteiner
Alternate: Colin Hufman

Women
 Canada	 China	 Denmark	 Great Britain	 Japan
Skip: Jennifer Jones
Third: Kaitlyn Lawes
Second: Jocelyn Peterman
Lead: Dawn McEwen
Alternate: Lisa Weagle

Skip: Han Yu
Third: Wang Rui
Second: Dong Ziqi
Lead: Zhang Lijun
Alternate: Jiang Xindi

Skip: Madeleine Dupont
Third: Mathilde Halse
Second: Denise Dupont
Lead: My Larsen
Alternate: Jasmin Lander

Skip: Eve Muirhead
Third: Vicky Wright
Second: Jennifer Dodds
Lead: Hailey Duff
Alternate: Mili Smith

Skip: Satsuki Fujisawa
Third: Chinami Yoshida
Second: Yumi Suzuki
Lead: Yurika Yoshida
Alternate: Kotomi Ishizaki

 ROC	 South Korea	 Sweden	 Switzerland	 United States
Skip: Alina Kovaleva
Third: Yulia Portunova
Second: Galina Arsenkina
Lead: Ekaterina Kuzmina
Alternate: Maria Komarova

Skip: Kim Eun-jung
Third: Kim Kyeong-ae
Second: Kim Cho-hi
Lead: Kim Seon-yeong
Alternate: Kim Yeong-mi

Skip: Anna Hasselborg
Third: Sara McManus
Second: Agnes Knochenhauer
Lead: Sofia Mabergs
Alternate: Johanna Heldin

Fourth: Alina Pätz
Skip: Silvana Tirinzoni
Second: Esther Neuenschwander
Lead: Melanie Barbezat
Alternate: Carole Howald

Skip: Tabitha Peterson
Third: Nina Roth
Second: Becca Hamilton
Lead: Tara Peterson
Alternate: Aileen Geving

Mixed doubles
 Australia	 Canada	 China	 Czech Republic	 Great Britain
Female: Tahli Gill
Male: Dean Hewitt

Female: Rachel Homan
Male: John Morris

Female: Fan Suyuan
Male: Ling Zhi

Female: Zuzana Paulová
Male: Tomáš Paul

Female: Jennifer Dodds
Male: Bruce Mouat

 Italy	 Norway	 Sweden	 Switzerland	 United States
Female: Stefania Constantini
Male: Amos Mosaner

Female: Kristin Skaslien
Male: Magnus Nedregotten

Female: Almida de Val
Male: Oskar Eriksson

Female: Jenny Perret
Male: Martin Rios

Female: Vicky Persinger
Male: Chris


key: _node_content
string_value:
{"id_": "5def565f-806e-4ec2-8a02-04b67a31256a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "047c47b0-7e16-43fa-a730-aa844eff6620", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "47ea804d505dd5ce008022d17dfca6ccc3e29db00b581bf64b2112c10572145e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "829c7ee4-fb09-4a27-a61e-3ed0d63c73a2", "node_type": "1", "metadata": {}, "hash": "a47273c7cc1564b52a9c3a76a3db579508580e08dd1b9c0486627c89a9a48e85", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14751, "end_char_idx": 17696, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Smith

Skip: Satsuki Fujisawa
Third: Chinami Yoshida
Second: Yumi Suzuki
Lead: Yurika Yoshida
Alternate: Kotomi Ishizaki

 ROC	 South Korea	 Sweden	 Switzerland	 United States
Skip: Alina Kovaleva
Third: Yulia Portunova
Second: Galina Arsenkina
Lead: Ekaterina Kuzmina
Alternate: Maria Komarova

Skip: Kim Eun-jung
Third: Kim Kyeong-ae
Second: Kim Cho-hi
Lead: Kim Seon-yeong
Alternate: Kim Yeong-mi

Skip: Anna Hasselborg
Third: Sara McManus
Second: Agnes Knochenhauer
Lead: Sofia Mabergs
Alternate: Johanna Heldin

Fourth: Alina Pätz
Skip: Silvana Tirinzoni
Second: Esther Neuenschwander
Lead: Melanie Barbezat
Alternate: Carole Howald

Skip: Tabitha Peterson
Third: Nina Roth
Second: Becca Hamilton
Lead: Tara Peterson
Alternate: Aileen Geving

Mixed doubles
 Australia	 Canada	 China	 Czech Republic	 Great Britain
Female: Tahli Gill
Male: Dean Hewitt

Female: Rachel Homan
Male: John Morris

Female: Fan Suyuan
Male: Ling Zhi

Female: Zuzana Paulová
Male: Tomáš Paul

Female: Jennifer Dodds
Male: Bruce Mouat

 Italy	 Norway	 Sweden	 Switzerland	 United States
Female: Stefania Constantini
Male: Amos Mosaner

Female: Kristin Skaslien
Male: Magnus Nedregotten

Female: Almida de Val
Male: Oskar Eriksson

Female: Jenny Perret
Male: Martin Rios

Female: Vicky Persinger
Male: Chris Plys
"""
```


```python
query = f"""Use the below article on the 2022 Winter Olympics to answer the subsequent question. If the answer cannot be found, write "I don't know."

Article:
\"\"\"
{wikipedia_article_on_curling}
\"\"\"

Question: Which athletes won the gold medal in curling at the 2022 Winter Olympics?"""

response = client.chat.completions.create(
    messages=[
        {'role': 'system', 'content': 'You answer questions about the 2022 Winter Olympics.'},
        {'role': 'user', 'content': query},
    ],
    model=GPT_MODEL,
    temperature=0,
)

print(response.choices[0].message.content)
```

    In the men's curling event, the gold medal was won by Sweden. In the women's curling event, the gold medal was won by Great Britain. In the mixed doubles curling event, the gold medal was won by Italy.


Thanks to the Wikipedia article included in the input message, GPT answers correctly.

In this particular case, GPT was intelligent enough to realize that the original question was underspecified, as there were three curling gold medal events, not just one.

Of course, this example partly relied on human intelligence. We knew the question was about curling, so we inserted a Wikipedia article on curling.

The rest of this notebook shows how to automate this knowledge insertion with embeddings-based search.

## 1. Prepare search data

To save you the time & expense, we've prepared a pre-embedded dataset of a few hundred Wikipedia articles about the 2022 Winter Olympics.

To see how we constructed this dataset, or to modify it yourself, see [Embedding Wikipedia articles for search](Embedding_Wikipedia_articles_for_search.ipynb).


key: _node_content
string_value:
{"id_": "829c7ee4-fb09-4a27-a61e-3ed0d63c73a2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5def565f-806e-4ec2-8a02-04b67a31256a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "aa7314b3053b337ee1fb22f3776475142bfef5651b21921bb54382ebc0b4e1fd", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2440359a-23ac-4901-87b2-d7ba26a5d368", "node_type": "1", "metadata": {}, "hash": "6a068a8cfe613597bafd33e7b2a8f51366cb6ae2aff9242a1b014f48520f06c4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Eriksson

Female: Jenny Perret
Male: Martin Rios

Female: Vicky Persinger
Male: Chris Plys
"""
```


```python
query = f"""Use the below article on the 2022 Winter Olympics to answer the subsequent question. If the answer cannot be found, write "I don't know."

Article:
\"\"\"
{wikipedia_article_on_curling}
\"\"\"

Question: Which athletes won the gold medal in curling at the 2022 Winter Olympics?"""

response = client.chat.completions.create(
    messages=[
        {'role': 'system', 'content': 'You answer questions about the 2022 Winter Olympics.'},
        {'role': 'user', 'content': query},
    ],
    model=GPT_MODEL,
    temperature=0,
)

print(response.choices[0].message.content)
```

    In the men's curling event, the gold medal was won by Sweden. In the women's curling event, the gold medal was won by Great Britain. In the mixed doubles curling event, the gold medal was won by Italy.


Thanks to the Wikipedia article included in the input message, GPT answers correctly.

In this particular case, GPT was intelligent enough to realize that the original question was underspecified, as there were three curling gold medal events, not just one.

Of course, this example partly relied on human intelligence. We knew the question was about curling, so we inserted a Wikipedia article on curling.

The rest of this notebook shows how to automate this knowledge insertion with embeddings-based search.

## 1. Prepare search data

To save you the time & expense, we've prepared a pre-embedded dataset of a few hundred Wikipedia articles about the 2022 Winter Olympics.

To see how we constructed this dataset, or to modify it yourself, see [Embedding Wikipedia articles for search](Embedding_Wikipedia_articles_for_search.ipynb).


```python
# download pre-chunked text and pre-computed embeddings
# this file is ~200 MB, so may take a minute depending on your connection speed
embeddings_path = "https://cdn.openai.com/API/examples/data/winter_olympics_2022.csv"

df = pd.read_csv(embeddings_path)
```


```python
# convert embeddings from CSV str type back to list type
df['embedding'] = df['embedding'].apply(ast.literal_eval)
```


```python
# the dataframe has two columns: "text" and "embedding"
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>text</th>
      <th>embedding</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n{{Oly.</td>
      <td>[-0.005021067801862955, 0.00026050032465718687.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==His.</td>
      <td>[0.0033927420154213905, -0.007447326090186834,


key: _node_content
string_value:
{"id_": "2440359a-23ac-4901-87b2-d7ba26a5d368", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "829c7ee4-fb09-4a27-a61e-3ed0d63c73a2", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "73bb23cdcfbe7e443b0117966401f3decd00af59acded24a84aaedd5196772ed", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "097229fb-7622-4b24-a0b4-40e8cb9435fb", "node_type": "1", "metadata": {}, "hash": "f8a637b7ab1902b00370e0c461027c08bb148b0f871a22fd80e4e168fe4b5771", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# download pre-chunked text and pre-computed embeddings
# this file is ~200 MB, so may take a minute depending on your connection speed
embeddings_path = "https://cdn.openai.com/API/examples/data/winter_olympics_2022.csv"

df = pd.read_csv(embeddings_path)
```


```python
# convert embeddings from CSV str type back to list type
df['embedding'] = df['embedding'].apply(ast.literal_eval)
```


```python
# the dataframe has two columns: "text" and "embedding"
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>text</th>
      <th>embedding</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n{{Oly.</td>
      <td>[-0.005021067801862955, 0.00026050032465718687.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==His.</td>
      <td>[0.0033927420154213905, -0.007447326090186834,</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==Ven.</td>
      <td>[-0.00915789045393467, -0.008366798982024193, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==Ven.</td>
      <td>[0.0030951891094446182, -0.006064314860850573,</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==Ven.</td>
      <td>[-0.002936174161732197, -0.006185177247971296,</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>6054</th>
      <td>Anaïs Chevalier-Bouchet\n\n==Personal life==\n.</td>
      <td>[-0.027750400826334953, 0.001746018067933619, .</td>
    </tr>
    <tr>
      <th>6055</th>
      <td>Uliana Nigmatullina\n\n{{short description|Rus.</td>
      <td>[-0.021714167669415474, 0.016001321375370026, .</td>
    </tr>
    <tr>
      <th>6056</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.


key: _node_content
string_value:
{"id_": "097229fb-7622-4b24-a0b4-40e8cb9435fb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2440359a-23ac-4901-87b2-d7ba26a5d368", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9e2d9518915dc5befeef724d57a24c6fb65927c0793a3198cedbb33e062f0661", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a258b507-5b73-4a90-9bc5-24f9c61d1ef5", "node_type": "1", "metadata": {}, "hash": "a3467c1edfed1cf7f7911d33e9fc4f5cf658f7a2eb9ce246dbb0f0e969f6eff3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</td>
      <td>[-0.00915789045393467, -0.008366798982024193, .</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==Ven.</td>
      <td>[0.0030951891094446182, -0.006064314860850573,</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Lviv bid for the 2022 Winter Olympics\n\n==Ven.</td>
      <td>[-0.002936174161732197, -0.006185177247971296,</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>6054</th>
      <td>Anaïs Chevalier-Bouchet\n\n==Personal life==\n.</td>
      <td>[-0.027750400826334953, 0.001746018067933619, .</td>
    </tr>
    <tr>
      <th>6055</th>
      <td>Uliana Nigmatullina\n\n{{short description|Rus.</td>
      <td>[-0.021714167669415474, 0.016001321375370026, .</td>
    </tr>
    <tr>
      <th>6056</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.029143543913960457, 0.014654331840574741, .</td>
    </tr>
    <tr>
      <th>6057</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.024266039952635765, 0.011665306985378265, .</td>
    </tr>
    <tr>
      <th>6058</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.021818075329065323, 0.005420385394245386, .</td>
    </tr>
  </tbody>
</table>
<p>6059 rows × 2 columns</p>
</div>



## 2. Search

Now we'll define a search function that:
- Takes a user query and a dataframe with text & embedding columns
- Embeds the user query with the OpenAI API
- Uses distance between query embedding and text embeddings to rank the texts
- Returns two lists:
    - The top N texts, ranked by relevance
    - Their corresponding relevance scores


key: _node_content
string_value:
{"id_": "a258b507-5b73-4a90-9bc5-24f9c61d1ef5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "097229fb-7622-4b24-a0b4-40e8cb9435fb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "faaa1a5960f8e9a43700b1d7bb0b893f0b1bb0caab30d94037d46296fa7f590a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "979421bb-6f17-4c97-81bd-f321f91a86b8", "node_type": "1", "metadata": {}, "hash": "5419d4dd49a3e61646150cfd67dc60dc0352ccdf42e7861239f29965ee64bcb4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
027750400826334953, 0.001746018067933619, .</td>
    </tr>
    <tr>
      <th>6055</th>
      <td>Uliana Nigmatullina\n\n{{short description|Rus.</td>
      <td>[-0.021714167669415474, 0.016001321375370026, .</td>
    </tr>
    <tr>
      <th>6056</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.029143543913960457, 0.014654331840574741, .</td>
    </tr>
    <tr>
      <th>6057</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.024266039952635765, 0.011665306985378265, .</td>
    </tr>
    <tr>
      <th>6058</th>
      <td>Uliana Nigmatullina\n\n==Biathlon results==\n\.</td>
      <td>[-0.021818075329065323, 0.005420385394245386, .</td>
    </tr>
  </tbody>
</table>
<p>6059 rows × 2 columns</p>
</div>



## 2. Search

Now we'll define a search function that:
- Takes a user query and a dataframe with text & embedding columns
- Embeds the user query with the OpenAI API
- Uses distance between query embedding and text embeddings to rank the texts
- Returns two lists:
    - The top N texts, ranked by relevance
    - Their corresponding relevance scores


```python
# search function
def strings_ranked_by_relatedness(
    query: str,
    df: pd.DataFrame,
    relatedness_fn=lambda x, y: 1 - spatial.distance.cosine(x, y),
    top_n: int = 100
) -> tuple[list[str], list[float]]:
    """Returns a list of strings and relatednesses, sorted from most related to least."""
    query_embedding_response = client.embeddings.create(
        model=EMBEDDING_MODEL,
        input=query,
    )
    query_embedding = query_embedding_response.data[0].embedding
    strings_and_relatednesses = [
        (row["text"], relatedness_fn(query_embedding, row["embedding"]))
        for i, row in df.iterrows()
    ]
    strings_and_relatednesses.sort(key=lambda x: x[1], reverse=True)
    strings, relatednesses = zip(*strings_and_relatednesses)
    return strings[:top_n], relatednesses[:top_n]

```


```python
# examples
strings, relatednesses = strings_ranked_by_relatedness("curling gold medal", df, top_n=5)
for string, relatedness in zip(strings, relatednesses):
    print(f"{relatedness=:.3f}")
    display(string)
```

    relatedness=0.879


key: _node_content
string_value:
{"id_": "979421bb-6f17-4c97-81bd-f321f91a86b8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a258b507-5b73-4a90-9bc5-24f9c61d1ef5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "acd3ddb6773a2771ec235735c4dfb7318ef689111f19fcb3f99ec4ed92f54df8", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4d8341e0-6e5b-47d5-bdb2-6337464f81ec", "node_type": "1", "metadata": {}, "hash": "df1b0adfbc78e798822c63a293df5bc7ac278c7379318c489dae2db590bf7292", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
021818075329065323, 0.005420385394245386, .</td>
    </tr>
  </tbody>
</table>
<p>6059 rows × 2 columns</p>
</div>



## 2. Search

Now we'll define a search function that:
- Takes a user query and a dataframe with text & embedding columns
- Embeds the user query with the OpenAI API
- Uses distance between query embedding and text embeddings to rank the texts
- Returns two lists:
    - The top N texts, ranked by relevance
    - Their corresponding relevance scores


```python
# search function
def strings_ranked_by_relatedness(
    query: str,
    df: pd.DataFrame,
    relatedness_fn=lambda x, y: 1 - spatial.distance.cosine(x, y),
    top_n: int = 100
) -> tuple[list[str], list[float]]:
    """Returns a list of strings and relatednesses, sorted from most related to least."""
    query_embedding_response = client.embeddings.create(
        model=EMBEDDING_MODEL,
        input=query,
    )
    query_embedding = query_embedding_response.data[0].embedding
    strings_and_relatednesses = [
        (row["text"], relatedness_fn(query_embedding, row["embedding"]))
        for i, row in df.iterrows()
    ]
    strings_and_relatednesses.sort(key=lambda x: x[1], reverse=True)
    strings, relatednesses = zip(*strings_and_relatednesses)
    return strings[:top_n], relatednesses[:top_n]

```


```python
# examples
strings, relatednesses = strings_ranked_by_relatedness("curling gold medal", df, top_n=5)
for string, relatedness in zip(strings, relatednesses):
    print(f"{relatedness=:.3f}")
    display(string)
```

    relatedness=0.879



    'Curling at the 2022 Winter Olympics\n\n==Medal summary==\n\n===Medal table===\n\n{{Medals table\n | caption        = \n | host           = \n | flag_template  = flagIOC\n | event          = 2022 Winter\n | team           = \n | gold_CAN = 0 | silver_CAN = 0 | bronze_CAN = 1\n | gold_ITA = 1 | silver_ITA = 0 | bronze_ITA = 0\n | gold_NOR = 0 | silver_NOR = 1 | bronze_NOR = 0\n | gold_SWE = 1 | silver_SWE = 0 | bronze_SWE = 2\n | gold_GBR = 1 | silver_GBR = 1 | bronze_GBR = 0\n | gold_JPN = 0 | silver_JPN = 1 | bronze_JPN - 0\n}}'


    relatedness=0.872


key: _node_content
string_value:
{"id_": "4d8341e0-6e5b-47d5-bdb2-6337464f81ec", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "979421bb-6f17-4c97-81bd-f321f91a86b8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "351de65a2fa8bbbf4349417ba975533ca16b9f6e37f71cd3acc1433e49a40b28", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2930ff1b-4002-4ef6-bd9e-f2a42c089e76", "node_type": "1", "metadata": {}, "hash": "0104dd96054a95b1f826e95c830e539a9fded271d63de0567cb6890aea2450e5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 21584, "end_char_idx": 23026, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# examples
strings, relatednesses = strings_ranked_by_relatedness("curling gold medal", df, top_n=5)
for string, relatedness in zip(strings, relatednesses):
    print(f"{relatedness=:.3f}")
    display(string)
```

    relatedness=0.879



    'Curling at the 2022 Winter Olympics\n\n==Medal summary==\n\n===Medal table===\n\n{{Medals table\n | caption        = \n | host           = \n | flag_template  = flagIOC\n | event          = 2022 Winter\n | team           = \n | gold_CAN = 0 | silver_CAN = 0 | bronze_CAN = 1\n | gold_ITA = 1 | silver_ITA = 0 | bronze_ITA = 0\n | gold_NOR = 0 | silver_NOR = 1 | bronze_NOR = 0\n | gold_SWE = 1 | silver_SWE = 0 | bronze_SWE = 2\n | gold_GBR = 1 | silver_GBR = 1 | bronze_GBR = 0\n | gold_JPN = 0 | silver_JPN = 1 | bronze_JPN - 0\n}}'


    relatedness=0.872



    "Curling at the 2022 Winter Olympics\n\n==Results summary==\n\n===Women's tournament===\n\n====Playoffs====\n\n=====Gold medal game=====\n\n''Sunday, 20 February, 9:05''\n{{#lst:Curling at the 2022 Winter Olympics – Women's tournament|GM}}\n{{Player percentages\n| team1 = {{flagIOC|JPN|2022 Winter}}\n| [[Yurika Yoshida]] | 97%\n| [[Yumi Suzuki]] | 82%\n| [[Chinami Yoshida]] | 64%\n| [[Satsuki Fujisawa]] | 69%\n| teampct1 = 78%\n| team2 = {{flagIOC|GBR|2022 Winter}}\n| [[Hailey Duff]] | 90%\n| [[Jennifer Dodds]] | 89%\n| [[Vicky Wright]] | 89%\n| [[Eve Muirhead]] | 88%\n| teampct2 = 89%\n}}"


    relatedness=0.869


key: _node_content
string_value:
{"id_": "2930ff1b-4002-4ef6-bd9e-f2a42c089e76", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4d8341e0-6e5b-47d5-bdb2-6337464f81ec", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4eab61e82ad7cc48d1a571a4bce99e63041e6cc930d533901a420246943c1bf0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a7455874-a70b-4a12-87ac-b61a75c28f7c", "node_type": "1", "metadata": {}, "hash": "1b3fbabac342866355082f15612faeb4c77f08f16b317aaa3851f6bd04465005", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 22380, "end_char_idx": 23738, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
relatedness=0.872



    "Curling at the 2022 Winter Olympics\n\n==Results summary==\n\n===Women's tournament===\n\n====Playoffs====\n\n=====Gold medal game=====\n\n''Sunday, 20 February, 9:05''\n{{#lst:Curling at the 2022 Winter Olympics – Women's tournament|GM}}\n{{Player percentages\n| team1 = {{flagIOC|JPN|2022 Winter}}\n| [[Yurika Yoshida]] | 97%\n| [[Yumi Suzuki]] | 82%\n| [[Chinami Yoshida]] | 64%\n| [[Satsuki Fujisawa]] | 69%\n| teampct1 = 78%\n| team2 = {{flagIOC|GBR|2022 Winter}}\n| [[Hailey Duff]] | 90%\n| [[Jennifer Dodds]] | 89%\n| [[Vicky Wright]] | 89%\n| [[Eve Muirhead]] | 88%\n| teampct2 = 89%\n}}"


    relatedness=0.869



    'Curling at the 2022 Winter Olympics\n\n==Results summary==\n\n===Mixed doubles tournament===\n\n====Playoffs====\n\n=====Gold medal game=====\n\n\'\'Tuesday, 8 February, 20:05\'\'\n{{#lst:Curling at the 2022 Winter Olympics – Mixed doubles tournament|GM}}\n{| class="wikitable"\n!colspan=4 width=400|Player percentages\n|-\n!colspan=2 width=200 style="white-space:nowrap;"| {{flagIOC|ITA|2022 Winter}}\n!colspan=2 width=200 style="white-space:nowrap;"| {{flagIOC|NOR|2022 Winter}}\n|-\n| [[Stefania Constantini]] || 83%\n| [[Kristin Skaslien]] || 70%\n|-\n| [[Amos Mosaner]] || 90%\n| [[Magnus Nedregotten]] || 69%\n|-\n| \'\'\'Total\'\'\' || 87%\n| \'\'\'Total\'\'\' || 69%\n|}'


    relatedness=0.868


key: _node_content
string_value:
{"id_": "a7455874-a70b-4a12-87ac-b61a75c28f7c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2930ff1b-4002-4ef6-bd9e-f2a42c089e76", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3e7f950bf26733ae18a9b20ef51925c817768bdae5f36973d3618536a41f30ac", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ce3c5a68-670e-4954-9424-ff35609d46d8", "node_type": "1", "metadata": {}, "hash": "e20460a7f68fdb89bb783784267d2359b33f725c583c84fc72cf80156b80bd09", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 23009, "end_char_idx": 25268, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
relatedness=0.869



    'Curling at the 2022 Winter Olympics\n\n==Results summary==\n\n===Mixed doubles tournament===\n\n====Playoffs====\n\n=====Gold medal game=====\n\n\'\'Tuesday, 8 February, 20:05\'\'\n{{#lst:Curling at the 2022 Winter Olympics – Mixed doubles tournament|GM}}\n{| class="wikitable"\n!colspan=4 width=400|Player percentages\n|-\n!colspan=2 width=200 style="white-space:nowrap;"| {{flagIOC|ITA|2022 Winter}}\n!colspan=2 width=200 style="white-space:nowrap;"| {{flagIOC|NOR|2022 Winter}}\n|-\n| [[Stefania Constantini]] || 83%\n| [[Kristin Skaslien]] || 70%\n|-\n| [[Amos Mosaner]] || 90%\n| [[Magnus Nedregotten]] || 69%\n|-\n| \'\'\'Total\'\'\' || 87%\n| \'\'\'Total\'\'\' || 69%\n|}'


    relatedness=0.868



    "Curling at the 2022 Winter Olympics\n\n==Medal summary==\n\n===Medalists===\n\n{| {{MedalistTable|type=Event|columns=1}}\n|-\n|Men<br/>{{DetailsLink|Curling at the 2022 Winter Olympics – Men's tournament}}\n|{{flagIOC|SWE|2022 Winter}}<br>[[Niklas Edin]]<br>[[Oskar Eriksson]]<br>[[Rasmus Wranå]]<br>[[Christoffer Sundgren]]<br>[[Daniel Magnusson (curler)|Daniel Magnusson]]\n|{{flagIOC|GBR|2022 Winter}}<br>[[Bruce Mouat]]<br>[[Grant Hardie]]<br>[[Bobby Lammie]]<br>[[Hammy McMillan Jr.]]<br>[[Ross Whyte]]\n|{{flagIOC|CAN|2022 Winter}}<br>[[Brad Gushue]]<br>[[Mark Nichols (curler)|Mark Nichols]]<br>[[Brett Gallant]]<br>[[Geoff Walker (curler)|Geoff Walker]]<br>[[Marc Kennedy]]\n|-\n|Women<br/>{{DetailsLink|Curling at the 2022 Winter Olympics – Women's tournament}}\n|{{flagIOC|GBR|2022 Winter}}<br>[[Eve Muirhead]]<br>[[Vicky Wright]]<br>[[Jennifer Dodds]]<br>[[Hailey Duff]]<br>[[Mili Smith]]\n|{{flagIOC|JPN|2022 Winter}}<br>[[Satsuki Fujisawa]]<br>[[Chinami Yoshida]]<br>[[Yumi Suzuki]]<br>[[Yurika Yoshida]]<br>[[Kotomi Ishizaki]]\n|{{flagIOC|SWE|2022 Winter}}<br>[[Anna Hasselborg]]<br>[[Sara McManus]]<br>[[Agnes Knochenhauer]]<br>[[Sofia Mabergs]]<br>[[Johanna Heldin]]\n|-\n|Mixed doubles<br/>{{DetailsLink|Curling at the 2022 Winter Olympics – Mixed doubles tournament}}\n|{{flagIOC|ITA|2022 Winter}}<br>[[Stefania Constantini]]<br>[[Amos Mosaner]]\n|{{flagIOC|NOR|2022 Winter}}<br>[[Kristin Skaslien]]<br>[[Magnus Nedregotten]]\n|{{flagIOC|SWE|2022 Winter}}<br>[[Almida de Val]]<br>[[Oskar Eriksson]]\n|}"


key: _node_content
string_value:
{"id_": "ce3c5a68-670e-4954-9424-ff35609d46d8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a7455874-a70b-4a12-87ac-b61a75c28f7c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ab3ca875f1ad4c47f87653dd096e31a4f27ead2bd3d81134ae86dda8a7eae00b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bdf22eac-983f-40ae-ad25-a9ecb58bc286", "node_type": "1", "metadata": {}, "hash": "897a437af892e0289ea0955e88756d4bd10759ae85f2e5c45133028ab7e31db1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 25275, "end_char_idx": 28139, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
relatedness=0.867



    "Curling at the 2022 Winter Olympics\n\n==Results summary==\n\n===Men's tournament===\n\n====Playoffs====\n\n=====Gold medal game=====\n\n''Saturday, 19 February, 14:50''\n{{#lst:Curling at the 2022 Winter Olympics – Men's tournament|GM}}\n{{Player percentages\n| team1 = {{flagIOC|GBR|2022 Winter}}\n| [[Hammy McMillan Jr.]] | 95%\n| [[Bobby Lammie]] | 80%\n| [[Grant Hardie]] | 94%\n| [[Bruce Mouat]] | 89%\n| teampct1 = 90%\n| team2 = {{flagIOC|SWE|2022 Winter}}\n| [[Christoffer Sundgren]] | 99%\n| [[Rasmus Wranå]] | 95%\n| [[Oskar Eriksson]] | 93%\n| [[Niklas Edin]] | 87%\n| teampct2 = 94%\n}}"


## 3. Ask

With the search function above, we can now automatically retrieve relevant knowledge and insert it into messages to GPT.

Below, we define a function `ask` that:
- Takes a user query
- Searches for text relevant to the query
- Stuffs that text into a message for GPT
- Sends the message to GPT
- Returns GPT's answer


```python
def num_tokens(text: str, model: str = GPT_MODEL) -> int:
    """Return the number of tokens in a string."""
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))


def query_message(
    query: str,
    df: pd.DataFrame,
    model: str,
    token_budget: int
) -> str:
    """Return a message for GPT, with relevant source texts pulled from a dataframe."""
    strings, relatednesses = strings_ranked_by_relatedness(query, df)
    introduction = 'Use the below articles on the 2022 Winter Olympics to answer the subsequent question. If the answer cannot be found in the articles, write "I could not find an answer."'
    question = f"\n\nQuestion: {query}"
    message = introduction
    for string in strings:
        next_article = f'\n\nWikipedia article section:\n"""\n{string}\n"""'
        if (
            num_tokens(message + next_article + question, model=model)
            > token_budget
        ):
            break
        else:
            message += next_article
    return message + question


def ask(
    query: str,
    df: pd.DataFrame = df,
    model: str = GPT_MODEL,
    token_budget: int = 4096 - 500,
    print_message: bool = False,
) -> str:
    """Answers a query using GPT and a dataframe of relevant texts and embeddings."""
    message = query_message(query, df, model=model, token_budget=token_budget)
    if print_message:
        print(message)
    messages = [
        {"role": "system", "content": "You answer questions about the 2022 Winter Olympics."},
        {"role": "user", "content": message},
    ]
    response = client.chat.completions.create(
        model=model,
        messages=messages,
        temperature=0
    )
    response_message = response.choices[0].message.content
    return response_message


```

### Example questions

Finally, let's ask our system our original question about gold medal curlers:


key: _node_content
string_value:
{"id_": "bdf22eac-983f-40ae-ad25-a9ecb58bc286", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a3cd2536-5fd0-4e47-a2e5-639393cbb918", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4ea8424201ef89e2522294baa2cc6ed0292d0cc4c38ad0a8b45721926466723e", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ce3c5a68-670e-4954-9424-ff35609d46d8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb", "title": "Question answering using embeddings-based search", "tokens": 13084, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "6fec4c6286c51793791bce4a6b223ea8ae8579d8aab4645004494790faae9e17", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "55517eca-96e9-4d2e-acc2-a9ab93893cc7", "node_type": "1", "metadata": {}, "hash": "58a788d33d23b19bd2bafb6213c58c8d8c6d63e68f9cd2617a3c8f81be38f761", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 26443, "end_char_idx": 29689, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


