# embeddings_queue

created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a9d81add-ec77-4661-abd5-ff106e462175
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a9d81add-ec77-4661-abd5-ff106e462175\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a5768f44-0abc-4c63-952e-1682a438f90b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"96972a3f3bfae8364f293157e7a22415d8d05b595b3c88b8148a595780820f68\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c02ccffc-786b-4b2f-b114-98cc9239fc74\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7a094f28bf2fcaa6a66a30c57cb265a28a20b4ac815915c71ac8b1f4a317b489\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21597, \"end_char_idx\": 24044, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">The main models we offer include:</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For completions: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-instruct, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For embeddings: text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-small, text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-large, and text-embedding-ada-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For fine-tuning jobs: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For moderations: text-moderation-stable and text-moderation.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Additionally, we have the latest models like gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f;"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c02ccffc-786b-4b2f-b114-98cc9239fc74
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c02ccffc-786b-4b2f-b114-98cc9239fc74\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a9d81add-ec77-4661-abd5-ff106e462175\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"b38daaa445f3249e38553be69062faf8a4a84eaf5b252c6765ef8956e93bd469\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3f1816d9-abcc-4257-94e4-c37ba5b2d8bd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e5be8d308086f9a8a5de4522146e583bf1481aeaa0abd179ddca9daaef9a213c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22835, \"end_char_idx\": 25163, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For fine-tuning jobs: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For moderations: text-moderation-stable and text-moderation.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Additionally, we have the latest models like gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Do you have a speech recognition model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 3f1816d9-abcc-4257-94e4-c37ba5b2d8bd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3f1816d9-abcc-4257-94e4-c37ba5b2d8bd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c02ccffc-786b-4b2f-b114-98cc9239fc74\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"7b48f8aecf4df586c9d6a84263f022bffea000636cecd596ef95d507bd7bc724\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6f024d8c-b62d-495c-9132-f2332822c0ba\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"de9f03a23d21939c8ee51742c77a36155acdc90fa6b6d8975f05fb7400d31c9c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23877, \"end_char_idx\": 26084, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Do you have a speech recognition model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6f024d8c-b62d-495c-9132-f2332822c0ba
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6f024d8c-b62d-495c-9132-f2332822c0ba\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3f1816d9-abcc-4257-94e4-c37ba5b2d8bd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c63fc06b32b2f4bbd69cc17ae81658ce47834e5c36e8abaa034a9d1e5a839279\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"49797d37-c323-421e-aa78-4d7a0048b357\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a15ed8af85624a429687178daef5c2c1a08c97797cee34d9556a4c8c6941ee1d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 25168, \"end_char_idx\": 27705, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">MODEL</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DE S CRIPTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">tts-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">New  Text-to-speech </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The latest tex...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 49797d37-c323-421e-aa78-4d7a0048b357
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"49797d37-c323-421e-aa78-4d7a0048b357\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6f024d8c-b62d-495c-9132-f2332822c0ba\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"18211ea07077b7649e7ae01f99e83a6f7a18d1a399a1df2c8e3977dfff9fd7ea\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"98dc1dc7-8039-4e82-aa31-84eafbe0d027\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"19fd98d37c86cb6136cacb44fe66023c9cf3b2a3ea0b9aea05098cae62526de6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 27708, \"end_char_idx\": 29595, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.44</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ENDP OINT</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DATA USED</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">FOR TRAINING</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DEFAULT</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">RETENTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ELIGIBLE FO...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 98dc1dc7-8039-4e82-aa31-84eafbe0d027
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"98dc1dc7-8039-4e82-aa31-84eafbe0d027\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"49797d37-c323-421e-aa78-4d7a0048b357\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"45c89448631927c5d470fbda5547ca5fae1d350f6b98b4ba9b164402f379ac1f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a26cc548-831f-43b2-9404-057551cedbaf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d86cdf3a11a6bb310918e5aab51b22ba7ca96924281cb1c3f4cdf92b859f1cc7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 29589, \"end_char_idx\": 32043, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, the Whisper model is a general-purpose speech recognition model mentioned in the content, capable of </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">multilingual speech recognition, speech translation, and language identification. The v2-large model, referred to </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">as </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"whisper-1\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, is available through an API and is optimized for faster performance.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which embedding model should I use for non-English use cases?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a26cc548-831f-43b2-9404-057551cedbaf
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a26cc548-831f-43b2-9404-057551cedbaf\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"98dc1dc7-8039-4e82-aa31-84eafbe0d027\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"20b27efa6aaf05b8795e44cc3d7da00051eaf1b721743e0e34e6e74bebbf3330\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ad50e971-357d-4a46-a987-0258023319bb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b5aff794b9108b65dde78bd3ca80ceda270860ff317ccbcde9269b5054f788bc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 30865, \"end_char_idx\": 32400, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.46</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ad50e971-357d-4a46-a987-0258023319bb
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ad50e971-357d-4a46-a987-0258023319bb\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a26cc548-831f-43b2-9404-057551cedbaf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"70ba63a72185e223dc0eb2152be19ac2ababb7a0f0e26af228cf9f4bbcbe5cdc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"96999881-08fe-47f3-a35b-31da212bea38\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cad3e2b0b03ca2adbb162077da1a846065b401d6069eaae2694bc1c9b666ef12\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 31484, \"end_char_idx\": 34021, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.46</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">MODEL</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DE S CRIPTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">tts-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">New  Text-to-speech </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The latest tex...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 96999881-08fe-47f3-a35b-31da212bea38
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"96999881-08fe-47f3-a35b-31da212bea38\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ad50e971-357d-4a46-a987-0258023319bb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"50d061b7279b236abdd807eeac47b57ae84d17b467134a0d33e884a2ca57228f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c35ab626-caf3-432e-96cd-8be88564a858\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0aac3eae59addae4e703accba947b48b023bc827ac42c7e85ffe75a09d41d92c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 34024, \"end_char_idx\": 35794, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.40</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Multilingual capabilities</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">GPT-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">4</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\"> outperforms both previous larg...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c35ab626-caf3-432e-96cd-8be88564a858
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c35ab626-caf3-432e-96cd-8be88564a858\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"96999881-08fe-47f3-a35b-31da212bea38\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"fe8c40853018bed9ee06c2f87cae881850071e51212794eb5235b79932491997\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e92a1866-2d77-44a6-a285-a1def69afe9d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"aa9077f13241d95863161de656c028f4d1a2e0aebe3671457fac577b59bde19e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 35788, \"end_char_idx\": 37795, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">For non-English use cases, you should use the </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"V3 large\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> embedding model, as it is described as the most capable </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">for both English and non-English tasks, with an output dimension of </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">,</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">072</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Can I introduce new knowledge in my LLM app using RAG?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e92a1866-2d77-44a6-a285-a1def69afe9d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e92a1866-2d77-44a6-a285-a1def69afe9d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c35ab626-caf3-432e-96cd-8be88564a858\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"1936b94236d2a213c52cf9817b2014b78dacfb6c99a36498f525eb90edd0cdce\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f1ee7469-1fd3-4348-8abc-d5374be937fb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9b2f55dd8c2c238cf83b7a7520048ec39b972a8e3633846abd80328cfa5196e9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 36817, \"end_char_idx\": 38852, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Can I introduce new knowledge in my LLM app using RAG?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Retrieve information to Augment the model\u2019s knowledge and Generate the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u201cWhat is y...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f1ee7469-1fd3-4348-8abc-d5374be937fb
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f1ee7469-1fd3-4348-8abc-d5374be937fb\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e92a1866-2d77-44a6-a285-a1def69afe9d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"d6fedc813325885a04d26334a49f16fb4f390520a39014efa4593f9b91cedf5d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dd4ccde5-c84c-4786-865b-f5c56b70a565\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f9eb8763ba491b4ae4c80a7b6363ff234a6dffa7a5f2de3d1857b5f5814db18a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 37800, \"end_char_idx\": 39892, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Retrieve information to Augment the model\u2019s knowledge and Generate the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u201cWhat is y...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to use RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Good for  \u2705</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Not good for  \u274c</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Introducing new information to the model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Teaching ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: dd4ccde5-c84c-4786-865b-f5c56b70a565
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"dd4ccde5-c84c-4786-865b-f5c56b70a565\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f1ee7469-1fd3-4348-8abc-d5374be937fb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"2b1bb21b3de80c25dfdb56468d9bfde6c5aedc8c4f25ff93d1334ad36657b303\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b037dd33-1d48-430b-aaf7-6333a9b49df1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b79afdf59a8853970e405a3d57258838df60ebabbe0c5c0d2f763efa99370f8e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 38857, \"end_char_idx\": 41017, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to use RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Good for  \u2705</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Not good for  \u274c</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Introducing new information to the model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Teaching ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Data preparation: augmenting content</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What does \u201cAugmentingcontent\u201d mean?</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Augmenti...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: b037dd33-1d48-430b-aaf7-6333a9b49df1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"b037dd33-1d48-430b-aaf7-6333a9b49df1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dd4ccde5-c84c-4786-865b-f5c56b70a565\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"2aacc2d97f7103a2bc2fd4fa4b978ab223afe8966f134fe952ef5b0f71a60591\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a37011a59e753a8bd76b2629edb02a625e2ecb35b443836a67aeeb82905146af\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 39886, \"end_char_idx\": 42238, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Data preparation: augmenting content</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What does \u201cAugmentingcontent\u201d mean?</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Augmenti...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> How many examples do I need to fine-tune a model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b037dd33-1d48-430b-aaf7-6333a9b49df1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"177ee2b3443b01c80137f449fcf9c122e6c96be44980e040707be3923bca0c0e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5c0debe7-9932-4a1c-9a28-29c7737f1a95\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"914aac184da76168151c5f196379d55ca66060490f4b231b01ea31cb7426e2f4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 41011, \"end_char_idx\": 43499, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> How many examples do I need to fine-tune a model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.68</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is Fine-tuning</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Public Model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training data</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tunedmodel</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning a model consists...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 5c0debe7-9932-4a1c-9a28-29c7737f1a95
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5c0debe7-9932-4a1c-9a28-29c7737f1a95\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"3fe3391864b70a2b8ca86c88f09bda1b292636cc18b731bd2963d2bbc1807da5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d690c483-ad6e-4d3f-897e-21f333467e1a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5f6fe07088436f8c23e3b357fde7ec514c72b794553450952763bc7d44ed6c4e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 42600, \"end_char_idx\": 44624, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is Fine-tuning</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Public Model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training data</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tunedmodel</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning a model consists...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.62</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to fine-tune</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning is good for:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Following a given format or tone for the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Proce...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d690c483-ad6e-4d3f-897e-21f333467e1a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d690c483-ad6e-4d3f-897e-21f333467e1a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5c0debe7-9932-4a1c-9a28-29c7737f1a95\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"3385df3b82998fbd5d23793b036014ca7488decf815c8392a6b3efae47e0817b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52d73bdb-e880-43f6-b4de-74ac764a1020\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"813a2099892ca6bd73f0843192ae159e6c4a25eab17eb08d9e60716b336f19f2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 43493, \"end_char_idx\": 45613, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.62</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to fine-tune</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning is good for:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Following a given format or tone for the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Proce...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Overview</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning involves adjusting theparameters of pre-trained models on aspeci\ufb01c dataset or t...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 52d73bdb-e880-43f6-b4de-74ac764a1020
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"52d73bdb-e880-43f6-b4de-74ac764a1020\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d690c483-ad6e-4d3f-897e-21f333467e1a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"9c382b2042127d1d17fa1f641e1422444299a2e7ffb5f956b5b6da25ad770267\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f6aeb219-c8c9-4a52-9303-af0d2e37ed6d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8bc9fa8ce34c4b65204984a98a574fe7924cdc583107ab9f09cf33e9683bf5f5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 44618, \"end_char_idx\": 47112, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Overview</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning involves adjusting theparameters of pre-trained models on aspeci\ufb01c dataset or t...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">We recommend using </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">50</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">100</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> examples for fine-tuning a model, even though the minimum is </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">10</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which metric can I use to evaluate a summarization task?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f6aeb219-c8c9-4a52-9303-af0d2e37ed6d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f6aeb219-c8c9-4a52-9303-af0d2e37ed6d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"52d73bdb-e880-43f6-b4de-74ac764a1020\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"eadba6ef4e18c5cd2ac19b92c5c7788295e338bd5f27fe251568001664294b23\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e4c79905-7592-4334-bb0d-74f6018ecf46\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"23b24f5646d126a18d73f7e318fad42e04cd46b401b6a23d47e75eb3067331cb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 46489, \"end_char_idx\": 48526, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which metric can I use to evaluate a summarization task?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ROUGE is a common metric for evaluating machine summariz...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e4c79905-7592-4334-bb0d-74f6018ecf46
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e4c79905-7592-4334-bb0d-74f6018ecf46\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f6aeb219-c8c9-4a52-9303-af0d2e37ed6d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"66552fdbd9486b66a2d806e1c1342b95ec73899fc55cc14af65e0f53ff5c55a2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f0bda61b-73e3-4f9b-8dd2-290130eac375\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"37be058a33be894c3ed59e93a7b5323b801b43e93f6f91c8b87d23bcfda110df\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 47474, \"end_char_idx\": 49498, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ROUGE is a common metric for evaluating machine summariz...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Component evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Subjective evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Compari...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f0bda61b-73e3-4f9b-8dd2-290130eac375
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f0bda61b-73e3-4f9b-8dd2-290130eac375\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e4c79905-7592-4334-bb0d-74f6018ecf46\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"cc8959cc8ee11e4957b8608eca1d57eabcb3fb45f2082b4719306c32f22cfa28\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"37adb2cd-8729-40ed-955f-121e02f22a37\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7b0386d7119179de071a83afe5d53fc654116e61ea983d1f66847886f79e47d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 48531, \"end_char_idx\": 50555, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Component evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Subjective evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Compari...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.48</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">BLEU score is another standard metric, this time focusin...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 37adb2cd-8729-40ed-955f-121e02f22a37
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"37adb2cd-8729-40ed-955f-121e02f22a37\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f0bda61b-73e3-4f9b-8dd2-290130eac375\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"046191cdd9e518f6a0d0edb7d41b3053a83262d524008a1e8bc97ea56d31ec59\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d30dbb8a-6424-4f11-8e11-93ed84be995c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d64ff46d20f88f1726d61f508953d8c11f8b54905fa9d997c4ea17018821e4c2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 49492, \"end_char_idx\": 51703, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.48</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">BLEU score is another standard metric, this time focusin...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">ROUGE is a common metric you can use to evaluate a summarization task.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>\n<span style=\"color: #af005f; text-decoration-color: #af005f\">ground truth.</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d30dbb8a-6424-4f11-8e11-93ed84be995c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d30dbb8a-6424-4f11-8e11-93ed84be995c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"37adb2cd-8729-40ed-955f-121e02f22a37\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"ac20df63878a4856a3359deb24ab3706381b0a4a86702a9f541b8458ac6ca213\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"21ae55db8ab4fe75d8d4c228c3166894aceaca747fd5523c99f1f54151f08420\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 50549, \"end_char_idx\": 52828, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">ROUGE is a common metric you can use to evaluate a summarization task.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>\n<span style=\"color: #af005f; text-decoration-color: #af005f\">ground truth.</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.60</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Our ground truth matches the predicted answer, so the evaluation passes!</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Eval...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d30dbb8a-6424-4f11-8e11-93ed84be995c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"b5b37d97d2654c190633b769f6b25abc4296f35dfa3571c0b2237bdf36f01108\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"97211244-1d04-4b01-bf52-36853a83348f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"390f2232ed3c77cd6474f0ef027e27972a09866bb451bc2ceade6168bd1fd7fa\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 51708, \"end_char_idx\": 53885, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.60</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Our ground truth matches the predicted answer, so the evaluation passes!</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Eval...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.59</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 97211244-1d04-4b01-bf52-36853a83348f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"97211244-1d04-4b01-bf52-36853a83348f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"e3b2eeb2334c2a274b200dc6d88b14d02268b8f22c8387cbd19535ceb5cc54da\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"06d75069-f6eb-4f39-b0de-53e40523a777\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ee47e9d7e2d52f10d301ada39b25dee618d51c492b9037c5df1b728a76863c70\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 52822, \"end_char_idx\": 54242, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.59</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 06d75069-f6eb-4f39-b0de-53e40523a777
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"06d75069-f6eb-4f39-b0de-53e40523a777\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"97211244-1d04-4b01-bf52-36853a83348f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"dc0179d3445632ab59ccee7fb7390e27bfac8a72ee4c88ce672e2658d11ed2aa\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c314e1ec-5c98-4532-8a93-4ab5acc8ac1b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e2796b03cc5e9a22146d789ca69b93e8a03a016d867cad6ab570ddb10c24641\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 53190, \"end_char_idx\": 55214, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What they\u2019re good for</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What to be aware of</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">A good sta...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c314e1ec-5c98-4532-8a93-4ab5acc8ac1b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c314e1ec-5c98-4532-8a93-4ab5acc8ac1b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"06d75069-f6eb-4f39-b0de-53e40523a777\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"4e28c927ad2fc669e97033fd3e6d089249065c890ed09daf48eb0e8c24c5dbb5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d1c961c2-db90-417a-bfca-d9b43895fcea\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2d613399576b84432d72fe3e3edb284de0bc4ac7d4badf03828a130325b0468c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 54247, \"end_char_idx\": 57652, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What they\u2019re good for</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What to be aware of</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">A good sta...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">The content provided is relevant and offers a detailed example for an evaluation process comparing to a ground </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">truth. Here's a concise explanation based on the content:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">In the given example, the evaluation process involves a question-and-answer scenario to verify the accuracy of </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">information retrieved by a tool or system in response to a query. The question posed is, </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"What is the population of</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Canada?\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> The ground truth, or the correct answer, is established as </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"The population of Canada in 2023 is 39,566,248</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">people.\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> A tool labeled </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"LLM\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> is then used to search for the answer, which predicts </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"The current population of </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Canada is 39,566,248 as of Tuesday, May 23, 2023.\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> This predicted answer matches the ground truth exactly, </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">indicating that the evaluation passes. This process demonstrates how an evaluation can be used to verify the </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">accuracy of information retrieved by a tool, comparing the predicted answer to the ground truth to ensure </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">correctness.</span>\n\n--------------"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d1c961c2-db90-417a-bfca-d9b43895fcea
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d1c961c2-db90-417a-bfca-d9b43895fcea\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"db6ae619-b33c-4db0-bb24-9433261dcfbc\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c314e1ec-5c98-4532-8a93-4ab5acc8ac1b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"72e35f0560759578ef7d94b7dcbe83d6cce08b02cf06768c1ecb5a4c4ce43e35\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 57655, \"end_char_idx\": 58851, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "ref_doc_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "chroma:document": "</pre>\n\n\n\n## Wrapping up\n\nIn this notebook, we have learned how to develop a basic RAG pipeline based on PDF documents. This includes:\n\n- How to parse pdf documents, taking slide decks and an export from an HTML page as examples, using a python library as well as GPT-4V to interpret the visuals\n- How to process the extracted content, clean it and chunk it into several pieces\n- How to embed the processed content using OpenAI embeddings\n- How to retrieve content that is relevant to an input query\n- How to use GPT-4-turbo to generate an answer using the retrieved content as context\n\nIf you want to explore further, consider these optimisations:\n\n- Playing around with the prompts provided as examples\n- Chunking the content further and adding metadata as context to each chunk\n- Adding rule-based filtering on the retrieval results or re-ranking results to surface to most relevant content\n\nYou can apply the techniques covered in this notebook to multiple use cases, such as assistants that can access your proprietary data, customer service or FAQ bots that can read from your internal policies, or anything that requires leveraging rich documents that would be better understood as images."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 787eb8d2-57ea-4c3e-9651-752be093e7de
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"787eb8d2-57ea-4c3e-9651-752be093e7de\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1e01053d-9de7-4cea-b339-6b0f73e9881f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e39f8626-06f4-43b5-b812-987468392af2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ceff499f9b12eb7e39fc76d174c27b66b86100964ea46e7dd7a7b3bcec56cba4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3493, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "ref_doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "chroma:document": "## Code search using embeddings\n\nThis notebook shows how Ada embeddings can be used to implement semantic code search. For this demonstration, we use our own [openai-python code repository](https://github.com/openai/openai-python). We implement a simple version of file parsing and extracting of functions from python files, which can be embedded, indexed, and queried.\n\n### Helper Functions\n\nWe first setup some simple parsing functions that allow us to extract important information from our codebase.\n\n\n```python\nimport pandas as pd\nfrom pathlib import Path\n\nDEF_PREFIXES = ['def ', 'async def ']\nNEWLINE = '\\n'\n\ndef get_function_name(code):\n    \"\"\"\n    Extract function name from a line beginning with 'def' or 'async def'.\n    \"\"\"\n    for prefix in DEF_PREFIXES:\n        if code.startswith(prefix):\n            return code[len(prefix): code.index('(')]\n\n\ndef get_until_no_space(all_lines, i):\n    \"\"\"\n    Get all lines until a line outside the function definition is found.\n    \"\"\"\n    ret = [all_lines[i]]\n    for j in range(i + 1, len(all_lines)):\n        if len(all_lines[j]) == 0 or all_lines[j][0] in [' ', '\\t', ')']:\n            ret.append(all_lines[j])\n        else:\n            break\n    return NEWLINE.join(ret)\n\n\ndef get_functions(filepath):\n    \"\"\"\n    Get all functions in a Python file.\n    \"\"\"\n    with open(filepath, 'r') as file:\n        all_lines = file.read().replace('\\r', NEWLINE).split(NEWLINE)\n        for i, l in enumerate(all_lines):\n            for prefix in DEF_PREFIXES:\n                if l.startswith(prefix):\n                    code = get_until_no_space(all_lines, i)\n                    function_name = get_function_name(code)\n                    yield {\n                        'code': code,\n                        'function_name': function_name,\n                        'filepath': filepath,\n                    }\n                    break\n\n\ndef extract_functions_from_repo(code_root):\n    \"\"\"\n    Extract all .py functions from the repository.\n    \"\"\"\n    code_files = list(code_root.glob('**/*.py'))\n\n    num_files = len(code_files)\n    print(f'Total number of .py files: {num_files}')\n\n    if num_files == 0:\n        print('Verify openai-python repo exists and code_root is set correctly.')\n        return None\n\n    all_funcs = [\n        func\n        for code_file in code_files\n        for func in get_functions(str(code_file))\n    ]\n\n    num_funcs = len(all_funcs)\n    print(f'Total number of functions extracted: {num_funcs}')\n\n    return all_funcs\n```\n\n# Data Loading\n\nWe'll first load the openai-python folder and extract the needed information using the functions we defined above.\n\n\n```python\n# Set user root directory to the 'openai-python' repository\nroot_dir = Path.home()\n\n# Assumes the 'openai-python' repository exists in the user's root directory\ncode_root = root_dir / 'openai-python'\n\n# Extract all functions from the repository\nall_funcs = extract_functions_from_repo(code_root)\n```\n\n    Total number of .py files: 51\n    Total number of functions extracted: 97\n\n\nNow that we have our content, we can pass the data to the `text-embedding-3-small` model and get back our vector embeddings.\n\n\n```python\nfrom utils.embeddings_utils import get_embedding\n\ndf = pd.DataFrame(all_funcs)\ndf['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, model='text-embedding-3-small'))\ndf['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))\ndf.to_csv(\"data/code_search_openai-python.csv\", index=False)\ndf.head()\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e39f8626-06f4-43b5-b812-987468392af2
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e39f8626-06f4-43b5-b812-987468392af2\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1e01053d-9de7-4cea-b339-6b0f73e9881f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"787eb8d2-57ea-4c3e-9651-752be093e7de\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3eb4c330b3a4446bbca3a1faa691fed60a5122f219013cc76ce1ae3818c72983\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"48706980-2642-4874-91fa-3822f254012d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"772462d4fab00bfd7cb7d9837b5a09685351944f0bc8a3dfbfb84c11d67b1283\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1883, \"end_char_idx\": 5101, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "ref_doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "chroma:document": "def extract_functions_from_repo(code_root):\n    \"\"\"\n    Extract all .py functions from the repository.\n    \"\"\"\n    code_files = list(code_root.glob('**/*.py'))\n\n    num_files = len(code_files)\n    print(f'Total number of .py files: {num_files}')\n\n    if num_files == 0:\n        print('Verify openai-python repo exists and code_root is set correctly.')\n        return None\n\n    all_funcs = [\n        func\n        for code_file in code_files\n        for func in get_functions(str(code_file))\n    ]\n\n    num_funcs = len(all_funcs)\n    print(f'Total number of functions extracted: {num_funcs}')\n\n    return all_funcs\n```\n\n# Data Loading\n\nWe'll first load the openai-python folder and extract the needed information using the functions we defined above.\n\n\n```python\n# Set user root directory to the 'openai-python' repository\nroot_dir = Path.home()\n\n# Assumes the 'openai-python' repository exists in the user's root directory\ncode_root = root_dir / 'openai-python'\n\n# Extract all functions from the repository\nall_funcs = extract_functions_from_repo(code_root)\n```\n\n    Total number of .py files: 51\n    Total number of functions extracted: 97\n\n\nNow that we have our content, we can pass the data to the `text-embedding-3-small` model and get back our vector embeddings.\n\n\n```python\nfrom utils.embeddings_utils import get_embedding\n\ndf = pd.DataFrame(all_funcs)\ndf['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, model='text-embedding-3-small'))\ndf['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))\ndf.to_csv(\"data/code_search_openai-python.csv\", index=False)\ndf.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>code</th>\n      <th>function_name</th>\n      <th>filepath</th>\n      <th>code_embedding</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>def _console_log_level():\\n    if openai.log i...</td>\n      <td>_console_log_level</td>\n      <td>openai/util.py</td>\n      <td>[0.005937571171671152, 0.05450401455163956, 0....</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>def log_debug(message, **params):\\n    msg = l...</td>\n      <td>log_debug</td>\n      <td>openai/util.py</td>\n      <td>[0.017557814717292786, 0.05647840350866318, -0...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>def log_info(message, **params):\\n    msg = lo...</td>\n      <td>log_info</td>\n      <td>openai/util.py</td>\n      <td>[0.022524144500494003, 0.06219055876135826, -0...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>def log_warn(message, **params):\\n    msg = lo...</td>\n      <td>log_warn</td>\n      <td>openai/util.py</td>\n      <td>[0.030524108558893204, 0.0667714849114418, -0....</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>def logfmt(props):\\n    def fmt(key, val):\\n  ...</td>\n      <td>logfmt</td>\n      <td>openai/util.py</td>\n      <td>[0.05337328091263771, 0.03697286546230316, -0....</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 48706980-2642-4874-91fa-3822f254012d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"48706980-2642-4874-91fa-3822f254012d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1e01053d-9de7-4cea-b339-6b0f73e9881f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e39f8626-06f4-43b5-b812-987468392af2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"044868805aa3c8153df80f541e7c9f691519660c7750903c2f07080b46571853\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fa51e041-9319-4639-93da-7945b3abe3d8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6b4275a7284796c3380c98c21c2edf7d70f67550d62a280412fa549c0657d8c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5105, \"end_char_idx\": 7982, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "ref_doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "chroma:document": "### Testing\n\nLet's test our endpoint with some simple queries. If you're familiar with the `openai-python` repository, you'll see that we're able to easily find functions we're looking for only a simple English description.\n\nWe define a search_functions method that takes our data that contains our embeddings, a query string, and some other configuration options. The process of searching our database works like such:\n\n1. We first embed our query string (code_query) with `text-embedding-3-small`. The reasoning here is that a query string like 'a function that reverses a string' and a function like 'def reverse(string): return string[::-1]' will be very similar when embedded.\n2. We then calculate the cosine similarity between our query string embedding and all data points in our database. This gives a distance between each point and our query.\n3. We finally sort all of our data points by their distance to our query string and return the number of results requested in the function parameters. \n\n\n```python\nfrom utils.embeddings_utils import cosine_similarity\n\ndef search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n    embedding = get_embedding(code_query, model='text-embedding-3-small')\n    df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n\n    res = df.sort_values('similarities', ascending=False).head(n)\n\n    if pprint:\n        for r in res.iterrows():\n            print(f\"{r[1].filepath}:{r[1].function_name}  score={round(r[1].similarities, 3)}\")\n            print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n            print('-' * 70)\n\n    return res\n```\n\n\n```python\nres = search_functions(df, 'fine-tuning input data validation logic', n=3)\n```\n\n    openai/validators.py:format_inferrer_validator  score=0.453\n    def format_inferrer_validator(df):\n        \"\"\"\n        This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n        It will also suggest to use ada and explain train/validation split benefits.\n        \"\"\"\n        ft_type = infer_task_type(df)\n        immediate_msg = None\n    ----------------------------------------------------------------------\n    openai/validators.py:infer_task_type  score=0.37\n    def infer_task_type(df):\n        \"\"\"\n        Infer the likely fine-tuning task type from the data\n        \"\"\"\n        CLASSIFICATION_THRESHOLD = 3  # min_average instances of each class\n        if sum(df.prompt.str.len()) == 0:\n            return \"open-ended generation\"\n    ----------------------------------------------------------------------\n    openai/validators.py:apply_validators  score=0.369\n    def apply_validators(\n        df,\n        fname,\n        remediation,\n        validators,\n        auto_accept,\n        write_out_file_func,\n    ----------------------------------------------------------------------"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: fa51e041-9319-4639-93da-7945b3abe3d8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fa51e041-9319-4639-93da-7945b3abe3d8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"1e01053d-9de7-4cea-b339-6b0f73e9881f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"48706980-2642-4874-91fa-3822f254012d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb\", \"title\": \"Data Loading\", \"tokens\": 2354, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b1f7f9518da7102af02084d5740fd90ca8141aefd49d17a5fe5cb10ff12c6cce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6733, \"end_char_idx\": 9958, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "ref_doc_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "chroma:document": "```python\nres = search_functions(df, 'fine-tuning input data validation logic', n=3)\n```\n\n    openai/validators.py:format_inferrer_validator  score=0.453\n    def format_inferrer_validator(df):\n        \"\"\"\n        This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n        It will also suggest to use ada and explain train/validation split benefits.\n        \"\"\"\n        ft_type = infer_task_type(df)\n        immediate_msg = None\n    ----------------------------------------------------------------------\n    openai/validators.py:infer_task_type  score=0.37\n    def infer_task_type(df):\n        \"\"\"\n        Infer the likely fine-tuning task type from the data\n        \"\"\"\n        CLASSIFICATION_THRESHOLD = 3  # min_average instances of each class\n        if sum(df.prompt.str.len()) == 0:\n            return \"open-ended generation\"\n    ----------------------------------------------------------------------\n    openai/validators.py:apply_validators  score=0.369\n    def apply_validators(\n        df,\n        fname,\n        remediation,\n        validators,\n        auto_accept,\n        write_out_file_func,\n    ----------------------------------------------------------------------\n\n\n\n```python\nres = search_functions(df, 'find common suffix', n=2, n_lines=10)\n```\n\n    openai/validators.py:get_common_xfix  score=0.487\n    def get_common_xfix(series, xfix=\"suffix\"):\n        \"\"\"\n        Finds the longest common suffix or prefix of all the values in a series\n        \"\"\"\n        common_xfix = \"\"\n        while True:\n            common_xfixes = (\n                series.str[-(len(common_xfix) + 1) :]\n                if xfix == \"suffix\"\n                else series.str[: len(common_xfix) + 1]\n    ----------------------------------------------------------------------\n    openai/validators.py:common_completion_suffix_validator  score=0.449\n    def common_completion_suffix_validator(df):\n        \"\"\"\n        This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n        \"\"\"\n        error_msg = None\n        immediate_msg = None\n        optional_msg = None\n        optional_fn = None\n    \n        ft_type = infer_task_type(df)\n    ----------------------------------------------------------------------\n\n\n\n```python\nres = search_functions(df, 'Command line interface for fine-tuning', n=1, n_lines=20)\n```\n\n    openai/cli.py:tools_register  score=0.391\n    def tools_register(parser):\n        subparsers = parser.add_subparsers(\n            title=\"Tools\", help=\"Convenience client side tools\"\n        )\n    \n        def help(args):\n            parser.print_help()\n    \n        parser.set_defaults(func=help)\n    \n        sub = subparsers.add_parser(\"fine_tunes.prepare_data\")\n        sub.add_argument(\n            \"-f\",\n            \"--file\",\n            required=True,\n            help=\"JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed.\"\n            \"This should be the local file path.\",\n        )\n        sub.add_argument(\n            \"-q\",\n    ----------------------------------------------------------------------"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0acb4008-3239-4b89-a8c8-a716ef862232
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0acb4008-3239-4b89-a8c8-a716ef862232\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b85c1bd9-db84-49d3-96c4-8df47b26d806\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"eb3e2b6df99bc6e6a26ace6c032b31afe81794e5040149be68e856459919748e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3265, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "## Named Entity Recognition (NER) to Enrich Text\n\n`Named Entity Recognition` (NER) is a `Natural Language Processing` task that identifies and classifies named entities (NE) into predefined semantic categories (such as persons, organizations, locations, events, time expressions, and quantities). By converting raw text into structured information, NER makes data more actionable, facilitating tasks like information extraction, data aggregation, analytics, and social media monitoring.\n\nThis notebook demonstrates how to carry out NER with [chat completion](https://platform.openai.com/docs/api-reference/chat) and [functions-calling](https://platform.openai.com/docs/guides/gpt/function-calling) to enrich a text with links to a knowledge base such as Wikipedia:\n\n**Text:**\n\n*In Germany, in 1440, goldsmith Johannes Gutenberg invented the movable-type printing press. His work led to an information revolution and the unprecedented mass-spread of literature throughout Europe. Modelled on the design of the existing screw presses, a single Renaissance movable-type printing press could produce up to 3,600 pages per workday.*\n\n**Text enriched with Wikipedia links:**\n\n*In [Germany](https://en.wikipedia.org/wiki/Germany), in 1440, goldsmith [Johannes Gutenberg]() invented the [movable-type printing press](https://en.wikipedia.org/wiki/Movable_Type). His work led to an [information revolution](https://en.wikipedia.org/wiki/Information_revolution) and the unprecedented mass-spread of literature throughout [Europe](https://en.wikipedia.org/wiki/Europe). Modelled on the design of the existing screw presses, a single [Renaissance](https://en.wikipedia.org/wiki/Renaissance) [movable-type printing press](https://en.wikipedia.org/wiki/Movable_Type) could produce up to 3,600 pages per workday.*\n\n**Inference Costs:** The notebook also illustrates how to estimate OpenAI API costs.\n\n### 1. Setup\n\n#### 1.1 Install/Upgrade Python packages\n\n\n```python\n%pip install --upgrade openai --quiet\n%pip install --upgrade nlpia2-wikipedia --quiet\n%pip install --upgrade tenacity --quiet\n```\n\n    Note: you may need to restart the kernel to use updated packages.\n    Note: you may need to restart the kernel to use updated packages.\n    Note: you may need to restart the kernel to use updated packages.\n\n\n#### 1.2 Load packages and OPENAI_API_KEY\n\nYou can generate an API key in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.\n\nThis notebook works with the latest OpeanAI models `gpt-3.5-turbo-0613` and `gpt-4-0613`.\n\n\n```python\nimport json\nimport logging\nimport os\n\nimport openai\nimport wikipedia\n\nfrom typing import Optional\nfrom IPython.display import display, Markdown\nfrom tenacity import retry, wait_random_exponential, stop_after_attempt\n\nlogging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')\n\nOPENAI_MODEL = 'gpt-3.5-turbo-0613'\n\nclient = openai.OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```\n\n### 2. Define the NER labels to be Identified\n\nWe define a standard set of NER labels to showcase a wide range of use cases. However, for our specific task of enriching text with knowledge base links, only a subset is practically required."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: b85c1bd9-db84-49d3-96c4-8df47b26d806
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"b85c1bd9-db84-49d3-96c4-8df47b26d806\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0acb4008-3239-4b89-a8c8-a716ef862232\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2760bb3dd3b350ed405caea5758dcbaed250b640a4b6ccc110b79f44646fa697\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"87698f77-bc26-4b5d-9cf1-828454540c2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4126aa6cf3da163ac65f4fe76981ffa7e1fbf9a10ddb43bd9d67518b61748c1b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1943, \"end_char_idx\": 5220, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\n%pip install --upgrade openai --quiet\n%pip install --upgrade nlpia2-wikipedia --quiet\n%pip install --upgrade tenacity --quiet\n```\n\n    Note: you may need to restart the kernel to use updated packages.\n    Note: you may need to restart the kernel to use updated packages.\n    Note: you may need to restart the kernel to use updated packages.\n\n\n#### 1.2 Load packages and OPENAI_API_KEY\n\nYou can generate an API key in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.\n\nThis notebook works with the latest OpeanAI models `gpt-3.5-turbo-0613` and `gpt-4-0613`.\n\n\n```python\nimport json\nimport logging\nimport os\n\nimport openai\nimport wikipedia\n\nfrom typing import Optional\nfrom IPython.display import display, Markdown\nfrom tenacity import retry, wait_random_exponential, stop_after_attempt\n\nlogging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')\n\nOPENAI_MODEL = 'gpt-3.5-turbo-0613'\n\nclient = openai.OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```\n\n### 2. Define the NER labels to be Identified\n\nWe define a standard set of NER labels to showcase a wide range of use cases. However, for our specific task of enriching text with knowledge base links, only a subset is practically required.\n\n\n```python\nlabels = [\n    \"person\",      # people, including fictional characters\n    \"fac\",         # buildings, airports, highways, bridges\n    \"org\",         # organizations, companies, agencies, institutions\n    \"gpe\",         # geopolitical entities like countries, cities, states\n    \"loc\",         # non-gpe locations\n    \"product\",     # vehicles, foods, appareal, appliances, software, toys \n    \"event\",       # named sports, scientific milestones, historical events\n    \"work_of_art\", # titles of books, songs, movies\n    \"law\",         # named laws, acts, or legislations\n    \"language\",    # any named language\n    \"date\",        # absolute or relative dates or periods\n    \"time\",        # time units smaller than a day\n    \"percent\",     # percentage (e.g., \"twenty percent\", \"18%\")\n    \"money\",       # monetary values, including unit\n    \"quantity\",    # measurements, e.g., weight or distance\n]\n```\n\n### 3. Prepare messages\n\nThe [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) takes a list of messages as input and delivers a model-generated message as an output. While the chat format is primarily designed for facilitating multi-turn conversations, it is equally efficient for single-turn tasks without any preceding conversation. For our purposes, we will specify a message for the system, assistant, and user roles.\n\n#### 3.1 System Message\n\nThe `system message` (prompt) sets the assistant's behavior by defining its desired persona and task. We also delineate the specific set of entity labels we aim to identify.\n\nAlthough one can instruct the model to format its response, it has to be noted that both `gpt-3.5-turbo-0613` and `gpt-4-0613` have been fine-tuned to discern when a function should be invoked, and to reply with `JSON` formatted according to the function's signature. This capability streamlines our prompt and enables us to receive structured data directly from the model."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 87698f77-bc26-4b5d-9cf1-828454540c2c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"87698f77-bc26-4b5d-9cf1-828454540c2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b85c1bd9-db84-49d3-96c4-8df47b26d806\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"105c2fbf95661a24dbe22fd64d0a2639b96fd5c11aeb7909d86f9a9330408a6a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"07144573-269a-44be-8a75-b7eab6da8331\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"36f625d5bbeadd68399cc4a41637eea392c7e93387bb5761832d3d0f9a703bf0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5223, \"end_char_idx\": 8183, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\ndef system_message(labels):\n    return f\"\"\"\nYou are an expert in Natural Language Processing. Your task is to identify common Named Entities (NER) in a given text.\nThe possible common Named Entities (NER) types are exclusively: ({\", \".join(labels)}).\"\"\"\n```\n\n#### 3.2 Assistant Message\n\n`Assistant messages` usually store previous assistant responses. However, as in our scenario, they can also be crafted to provide examples of the desired behavior. While OpenAI is able to execute `zero-shot` Named Entity Recognition, we have found that a `one-shot` approach produces more precise results.\n\n\n```python\ndef assisstant_message():\n    return f\"\"\"\nEXAMPLE:\n    Text: 'In Germany, in 1440, goldsmith Johannes Gutenberg invented the movable-type printing press. His work led to an information revolution and the unprecedented mass-spread / \n    of literature throughout Europe. Modelled on the design of the existing screw presses, a single Renaissance movable-type printing press could produce up to 3,600 pages per workday.'\n    {{\n        \"gpe\": [\"Germany\", \"Europe\"],\n        \"date\": [\"1440\"],\n        \"person\": [\"Johannes Gutenberg\"],\n        \"product\": [\"movable-type printing press\"],\n        \"event\": [\"Renaissance\"],\n        \"quantity\": [\"3,600 pages\"],\n        \"time\": [\"workday\"]\n    }}\n--\"\"\"\n```\n\n#### 3.3 User Message\n\nThe `user message` provides the specific text for the assistant task:\n\n\n```python\ndef user_message(text):\n    return f\"\"\"\nTASK:\n    Text: {text}\n\"\"\"\n```\n\n### 4. OpenAI Functions (and Utils)\n\nIn an OpenAI API call, we can describe `functions` to `gpt-3.5-turbo-0613` and `gpt-4-0613` and have the model intelligently choose to output a `JSON` object containing arguments to call those `functions`. It's important to note that the [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) doesn't actually execute the `function`. Instead, it provides the `JSON` output, which can then be used to call the `function` in our code. For more details, refer to the [OpenAI Function Calling Guide](https://platform.openai.com/docs/guides/function-calling).\n\nOur function, `enrich_entities(text, label_entities)` gets a block of text and a dictionary containing identified labels and entities as parameters. It then associates the recognized entities with their corresponding links to the Wikipedia articles.\n\n\n```python\n@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))\ndef find_link(entity: str) -> Optional[str]:\n    \"\"\"\n    Finds a Wikipedia link for a given entity.\n    \"\"\"\n    try:\n        titles = wikipedia.search(entity)\n        if titles:\n            # naively consider the first result as the best\n            page = wikipedia.page(titles[0])\n            return page.url\n    except (wikipedia.exceptions.WikipediaException) as ex:\n        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')\n\n    return None\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 07144573-269a-44be-8a75-b7eab6da8331
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"07144573-269a-44be-8a75-b7eab6da8331\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"87698f77-bc26-4b5d-9cf1-828454540c2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"849af42ad5b2866346a11f8e19d4f5a3b115495153671468e226f1beae21bfae\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7dbb4709-0527-4f6f-a161-18cb141f55e8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7660c9d6abb17a41374d6eba5ce131692ac72e9cc6b32a48df624513bf3a5e99\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6634, \"end_char_idx\": 8605, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\ndef user_message(text):\n    return f\"\"\"\nTASK:\n    Text: {text}\n\"\"\"\n```\n\n### 4. OpenAI Functions (and Utils)\n\nIn an OpenAI API call, we can describe `functions` to `gpt-3.5-turbo-0613` and `gpt-4-0613` and have the model intelligently choose to output a `JSON` object containing arguments to call those `functions`. It's important to note that the [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) doesn't actually execute the `function`. Instead, it provides the `JSON` output, which can then be used to call the `function` in our code. For more details, refer to the [OpenAI Function Calling Guide](https://platform.openai.com/docs/guides/function-calling).\n\nOur function, `enrich_entities(text, label_entities)` gets a block of text and a dictionary containing identified labels and entities as parameters. It then associates the recognized entities with their corresponding links to the Wikipedia articles.\n\n\n```python\n@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))\ndef find_link(entity: str) -> Optional[str]:\n    \"\"\"\n    Finds a Wikipedia link for a given entity.\n    \"\"\"\n    try:\n        titles = wikipedia.search(entity)\n        if titles:\n            # naively consider the first result as the best\n            page = wikipedia.page(titles[0])\n            return page.url\n    except (wikipedia.exceptions.WikipediaException) as ex:\n        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')\n\n    return None\n```\n\n\n```python\ndef find_all_links(label_entities:dict) -> dict:\n    \"\"\" \n    Finds all Wikipedia links for the dictionary entities in the whitelist label list.\n    \"\"\"\n    whitelist = ['event', 'gpe', 'org', 'person', 'product', 'work_of_art']\n    \n    return {e: find_link(e) for label, entities in label_entities.items() \n                            for e in entities\n                            if label in whitelist}\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 7dbb4709-0527-4f6f-a161-18cb141f55e8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7dbb4709-0527-4f6f-a161-18cb141f55e8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"07144573-269a-44be-8a75-b7eab6da8331\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"70788201670f03b5f20df3f882f3e5664a879f0807d71f4db74e90a7ca4046ab\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fbb8a2f1-b476-4797-98d2-376b91545b95\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"98cd09c61cfd0ec9e2b691ae4b7af41615e139b060f18bc4a839bb995b36a977\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7595, \"end_char_idx\": 10365, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\n@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))\ndef find_link(entity: str) -> Optional[str]:\n    \"\"\"\n    Finds a Wikipedia link for a given entity.\n    \"\"\"\n    try:\n        titles = wikipedia.search(entity)\n        if titles:\n            # naively consider the first result as the best\n            page = wikipedia.page(titles[0])\n            return page.url\n    except (wikipedia.exceptions.WikipediaException) as ex:\n        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')\n\n    return None\n```\n\n\n```python\ndef find_all_links(label_entities:dict) -> dict:\n    \"\"\" \n    Finds all Wikipedia links for the dictionary entities in the whitelist label list.\n    \"\"\"\n    whitelist = ['event', 'gpe', 'org', 'person', 'product', 'work_of_art']\n    \n    return {e: find_link(e) for label, entities in label_entities.items() \n                            for e in entities\n                            if label in whitelist}\n```\n\n\n```python\ndef enrich_entities(text: str, label_entities: dict) -> str:\n    \"\"\"\n    Enriches text with knowledge base links.\n    \"\"\"\n    entity_link_dict = find_all_links(label_entities)\n    logging.info(f\"entity_link_dict: {entity_link_dict}\")\n    \n    for entity, link in entity_link_dict.items():\n        text = text.replace(entity, f\"[{entity}]({link})\")\n\n    return text\n```\n\n### 4. ChatCompletion\n\nAs previously highlighted, `gpt-3.5-turbo-0613` and `gpt-4-0613` have been fine-tuned to detect when a `function` should to be called. Moreover, they can produce a `JSON` response that conforms to the `function` signature. Here's the sequence we follow:\n\n1. Define our `function` and its associated `JSON` Schema.\n2. Invoke the model using the `messages`, `tools` and `tool_choice` parameters.\n3. Convert the output into a `JSON` object, and then call the `function` with the `arguments` provided by the model.\n\nIn practice, one might want to re-invoke the model again by appending the `function` response as a new message, and let the model summarize the results back to the user. Nevertheless, for our purposes, this step is not needed.\n\n*Note that in a real-case scenario it is strongly recommended to build in user confirmation flows before taking actions.*\n\n#### 4.1 Define our Function and JSON schema\n\nSince we want the model to output a dictionary of labels and recognized entities:\n\n```python\n{   \n    \"gpe\": [\"Germany\", \"Europe\"],   \n    \"date\": [\"1440\"],   \n    \"person\": [\"Johannes Gutenberg\"],   \n    \"product\": [\"movable-type printing press\"],   \n    \"event\": [\"Renaissance\"],   \n    \"quantity\": [\"3,600 pages\"],   \n    \"time\": [\"workday\"]   \n}   \n```\nwe need to define the corresponding `JSON` schema to be passed to the `tools` parameter:"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: fbb8a2f1-b476-4797-98d2-376b91545b95
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fbb8a2f1-b476-4797-98d2-376b91545b95\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7dbb4709-0527-4f6f-a161-18cb141f55e8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"adb929e5c5784d61db7f9faf96bbbd5fc37610e4c15646aad313d9468c689006\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3617e664-073e-4d02-b405-14fe68f6a451\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"fd52f3aa36c997a614dd853d9b03ce869acc18b5f884cfec05b4c1c49ab5ecc6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10369, \"end_char_idx\": 12927, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\ndef generate_functions(labels: dict) -> list:\n    return [\n        {   \n            \"type\": \"function\",\n            \"function\": {\n                \"name\": \"enrich_entities\",\n                \"description\": \"Enrich Text with Knowledge Base Links\",\n                \"parameters\": {\n                    \"type\": \"object\",\n                        \"properties\": {\n                            \"r'^(?:' + '|'.join({labels}) + ')$'\": \n                            {\n                                \"type\": \"array\",\n                                \"items\": {\n                                    \"type\": \"string\"\n                                }\n                            }\n                        },\n                        \"additionalProperties\": False\n                },\n            }\n        }\n    ]\n```\n\n#### 4.2 Chat Completion\n\nNow, we invoke the model. It's important to note that we direct the API to use a specific function by setting the `tool_choice` parameter to `{\"type\": \"function\", \"function\" : {\"name\": \"enrich_entities\"}}`.\n\n\n```python\n@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))\ndef run_openai_task(labels, text):\n    messages = [\n          {\"role\": \"system\", \"content\": system_message(labels=labels)},\n          {\"role\": \"assistant\", \"content\": assisstant_message()},\n          {\"role\": \"user\", \"content\": user_message(text=text)}\n      ]\n\n    # TODO: functions and function_call are deprecated, need to be updated\n    # See: https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools\n    response = openai.chat.completions.create(\n        model=\"gpt-3.5-turbo-0613\",\n        messages=messages,\n        tools=generate_functions(labels),\n        tool_choice={\"type\": \"function\", \"function\" : {\"name\": \"enrich_entities\"}}, \n        temperature=0,\n        frequency_penalty=0,\n        presence_penalty=0,\n    )\n\n    response_message = response.choices[0].message\n    \n    available_functions = {\"enrich_entities\": enrich_entities}  \n    function_name = response_message.tool_calls[0].function.name\n    \n    function_to_call = available_functions[function_name]\n    logging.info(f\"function_to_call: {function_to_call}\")\n\n    function_args = json.loads(response_message.tool_calls[0].function.arguments)\n    logging.info(f\"function_args: {function_args}\")\n\n    function_response = function_to_call(text, function_args)\n\n    return {\"model_response\": response, \n            \"function_response\": function_response}\n```\n\n### 5. Let's Enrich a Text with Wikipedia links\n\n#### 5.1 Run OpenAI Task"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 3617e664-073e-4d02-b405-14fe68f6a451
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3617e664-073e-4d02-b405-14fe68f6a451\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fbb8a2f1-b476-4797-98d2-376b91545b95\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cab0d4263f7340b738188bec7e3df4bc0147f148a8bfe53f4d4b8394bc9e8356\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8d7ddbad-9bfe-41c6-ad65-61fec2588b37\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f3612cf294d02603f04246cab52e142f369399fe139adb5c8d3869dd84023926\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11411, \"end_char_idx\": 14027, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\n@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))\ndef run_openai_task(labels, text):\n    messages = [\n          {\"role\": \"system\", \"content\": system_message(labels=labels)},\n          {\"role\": \"assistant\", \"content\": assisstant_message()},\n          {\"role\": \"user\", \"content\": user_message(text=text)}\n      ]\n\n    # TODO: functions and function_call are deprecated, need to be updated\n    # See: https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools\n    response = openai.chat.completions.create(\n        model=\"gpt-3.5-turbo-0613\",\n        messages=messages,\n        tools=generate_functions(labels),\n        tool_choice={\"type\": \"function\", \"function\" : {\"name\": \"enrich_entities\"}}, \n        temperature=0,\n        frequency_penalty=0,\n        presence_penalty=0,\n    )\n\n    response_message = response.choices[0].message\n    \n    available_functions = {\"enrich_entities\": enrich_entities}  \n    function_name = response_message.tool_calls[0].function.name\n    \n    function_to_call = available_functions[function_name]\n    logging.info(f\"function_to_call: {function_to_call}\")\n\n    function_args = json.loads(response_message.tool_calls[0].function.arguments)\n    logging.info(f\"function_args: {function_args}\")\n\n    function_response = function_to_call(text, function_args)\n\n    return {\"model_response\": response, \n            \"function_response\": function_response}\n```\n\n### 5. Let's Enrich a Text with Wikipedia links\n\n#### 5.1 Run OpenAI Task\n\n\n```python\ntext = \"\"\"The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr.\"\"\"\nresult = run_openai_task(labels, text)\n```\n\n     2023-10-20 18:05:51,729 - INFO - function_to_call: <function enrich_entities at 0x0000021D30C462A0>\n     2023-10-20 18:05:51,730 - INFO - function_args: {'person': ['John Lennon', 'Paul McCartney', 'George Harrison', 'Ringo Starr'], 'org': ['The Beatles'], 'gpe': ['Liverpool'], 'date': ['1960']}\n     2023-10-20 18:06:09,858 - INFO - entity_link_dict: {'John Lennon': 'https://en.wikipedia.org/wiki/John_Lennon', 'Paul McCartney': 'https://en.wikipedia.org/wiki/Paul_McCartney', 'George Harrison': 'https://en.wikipedia.org/wiki/George_Harrison', 'Ringo Starr': 'https://en.wikipedia.org/wiki/Ringo_Starr', 'The Beatles': 'https://en.wikipedia.org/wiki/The_Beatles', 'Liverpool': 'https://en.wikipedia.org/wiki/Liverpool'}\n\n\n#### 5.2 Function Response\n\n\n```python\ndisplay(Markdown(f\"\"\"**Text:** {text}   \n                     **Enriched_Text:** {result['function_response']}\"\"\"))\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 8d7ddbad-9bfe-41c6-ad65-61fec2588b37
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"8d7ddbad-9bfe-41c6-ad65-61fec2588b37\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"13dd6230-0b7a-49b0-aa23-5f419986978c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3617e664-073e-4d02-b405-14fe68f6a451\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb\", \"title\": \"estimate inference cost assuming gpt-3.5-turbo (4K context)\", \"tokens\": 3604, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"66d69687369b9348514cb98f52d1b2cc2c59ba3d7e4e274793719f566462f2ce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12930, \"end_char_idx\": 15361, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "ref_doc_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "chroma:document": "```python\ntext = \"\"\"The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr.\"\"\"\nresult = run_openai_task(labels, text)\n```\n\n     2023-10-20 18:05:51,729 - INFO - function_to_call: <function enrich_entities at 0x0000021D30C462A0>\n     2023-10-20 18:05:51,730 - INFO - function_args: {'person': ['John Lennon', 'Paul McCartney', 'George Harrison', 'Ringo Starr'], 'org': ['The Beatles'], 'gpe': ['Liverpool'], 'date': ['1960']}\n     2023-10-20 18:06:09,858 - INFO - entity_link_dict: {'John Lennon': 'https://en.wikipedia.org/wiki/John_Lennon', 'Paul McCartney': 'https://en.wikipedia.org/wiki/Paul_McCartney', 'George Harrison': 'https://en.wikipedia.org/wiki/George_Harrison', 'Ringo Starr': 'https://en.wikipedia.org/wiki/Ringo_Starr', 'The Beatles': 'https://en.wikipedia.org/wiki/The_Beatles', 'Liverpool': 'https://en.wikipedia.org/wiki/Liverpool'}\n\n\n#### 5.2 Function Response\n\n\n```python\ndisplay(Markdown(f\"\"\"**Text:** {text}   \n                     **Enriched_Text:** {result['function_response']}\"\"\"))\n```\n\n\n**Text:** The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr.   \n                     **Enriched_Text:** [The Beatles](https://en.wikipedia.org/wiki/The_Beatles) were an English rock band formed in [Liverpool](https://en.wikipedia.org/wiki/Liverpool) in 1960, comprising [John Lennon](https://en.wikipedia.org/wiki/John_Lennon), [Paul McCartney](https://en.wikipedia.org/wiki/Paul_McCartney), [George Harrison](https://en.wikipedia.org/wiki/George_Harrison), and [Ringo Starr](https://en.wikipedia.org/wiki/Ringo_Starr).\n\n\n#### 5.3 Token Usage\n\nTo estimate the inference costs, we can parse the response's \"usage\" field. Detailed token costs per model are available in the [OpenAI Pricing Guide](https://openai.com/pricing):\n\n\n```python\n# estimate inference cost assuming gpt-3.5-turbo (4K context)\ni_tokens  = result[\"model_response\"].usage.prompt_tokens \no_tokens = result[\"model_response\"].usage.completion_tokens \n\ni_cost = (i_tokens / 1000) * 0.0015\no_cost = (o_tokens / 1000) * 0.002\n\nprint(f\"\"\"Token Usage\n    Prompt: {i_tokens} tokens\n    Completion: {o_tokens} tokens\n    Cost estimation: ${round(i_cost + o_cost, 5)}\"\"\")\n```\n\n    Token Usage\n        Prompt: 331 tokens\n        Completion: 47 tokens\n        Cost estimation: $0.00059"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 3fcc1241-17d9-48d8-8246-888400b00ff8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/README.ipynb", "title": "Vector Databases", "tokens": 586, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3fcc1241-17d9-48d8-8246-888400b00ff8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/README.ipynb\", \"title\": \"Vector Databases\", \"tokens\": 586, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"8555a529-4c1f-47b9-a733-69aee759e837\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/README.ipynb\", \"title\": \"Vector Databases\", \"tokens\": 586, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"57292be6e3afabcdaa5c7271b2922fab4af57c93d1fb1a265767a40d9c0c83b5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2256, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "8555a529-4c1f-47b9-a733-69aee759e837", "doc_id": "8555a529-4c1f-47b9-a733-69aee759e837", "ref_doc_id": "8555a529-4c1f-47b9-a733-69aee759e837", "chroma:document": "# Vector Databases\n\nThis section of the OpenAI Cookbook showcases many of the vector databases available to support your semantic search use cases.\n\nVector databases can be a great accompaniment for knowledge retrieval applications, which reduce hallucinations by providing the LLM with the relevant context to answer questions.\n\nEach provider has their own named directory, with a standard notebook to introduce you to using our API with their product, and any supplementary notebooks they choose to add to showcase their functionality.\n\n## Guides & deep dives\n- [AnalyticDB](https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/latest/get-started-with-analyticdb-for-postgresql)\n- [Cassandra/Astra DB](https://docs.datastax.com/en/astra-serverless/docs/vector-search/qandasimsearch-quickstart.html)\n- [Azure AI Search](https://learn.microsoft.com/azure/search/search-get-started-vector)\n- [Azure SQL Database](https://learn.microsoft.com/azure/azure-sql/database/ai-artificial-intelligence-intelligent-applications?view=azuresql)\n- [Chroma](https://docs.trychroma.com/getting-started)\n- [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html)\n- [Hologres](https://www.alibabacloud.com/help/en/hologres/latest/procedure-to-use-hologres)\n- [Kusto](https://learn.microsoft.com/en-us/azure/data-explorer/web-query-data)\n- [Milvus](https://milvus.io/docs/example_code.md)\n- [MyScale](https://docs.myscale.com/en/quickstart/)\n- [MongoDB](https://www.mongodb.com/products/platform/atlas-vector-search)\n- [Neon Postgres](https://neon.tech/docs/ai/ai-intro)\n- [Pinecone](https://docs.pinecone.io/docs/quickstart)\n- [PolarDB](https://www.alibabacloud.com/help/en/polardb/latest/quick-start)\n- [Qdrant](https://qdrant.tech/documentation/quick-start/)\n- [Redis](https://github.com/RedisVentures/simple-vecsim-intro)\n- [SingleStoreDB](https://www.singlestore.com/blog/how-to-get-started-with-singlestore/)\n- [Supabase](https://supabase.com/docs/guides/ai)\n- [Tembo](https://tembo.io/docs/product/stacks/ai/vectordb)\n- [Typesense](https://typesense.org/docs/guide/)\n- [Vespa AI](https://vespa.ai/)\n- [Weaviate](https://weaviate.io/developers/weaviate/quickstart)\n- [Zilliz](https://docs.zilliz.com/docs/quick-start-1)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e7190cd5-797f-491c-95b9-dae4b65c71fd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e7190cd5-797f-491c-95b9-dae4b65c71fd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7a7d7160-1636-4a30-887e-08d518422e34\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6b41a7a857a0b428bd46502f5bc5e40174f60e462e440a2d7d51cac8d7368019\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2225, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "# Kusto as a Vector database for AI embeddings\n\nThis Notebook provides step by step instuctions on using Azure Data Explorer (Kusto) as a vector database with OpenAI embeddings. \n\nThis notebook presents an end-to-end process of:\n\n1. Using precomputed embeddings created by OpenAI API.\n2. Storing the embeddings in Kusto.\n3. Converting raw text query to an embedding with OpenAI API.\n4. Using Kusto to perform cosine similarity search in the stored embeddings\n\n\n### Prerequisites\n\nFor the purposes of this exercise we need to prepare a couple of things:\n\n1. Azure Data Explorer(Kusto) server instance. https://azure.microsoft.com/en-us/products/data-explorer\n3. Azure OpenAI credentials or OpenAI API key.\n\n\n```python\n%pip install wget\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)\n\n\n\n\n\n\n\n    Collecting wget\n      Downloading wget-3.2.zip (10 kB)\n      Preparing metadata (setup.py) ... \u001b[?25ldone\n    \u001b[?25hBuilding wheels for collected packages: wget\n      Building wheel for wget (setup.py) ... \u001b[?25l-\b \bdone\n    \u001b[?25h  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9657 sha256=10fd8aa1d20fd49c36389dc888acc721d0578c5a0635fc9fc5dc642c0f49522e\n      Stored in directory: /home/trusted-service-user/.cache/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769\n    Successfully built wget\n    Installing collected packages: wget\n    Successfully installed wget-3.2\n    \n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip\u001b[0m\n    Note: you may need to restart the kernel to use updated packages.\n\n\n\n\n\n\n\n    Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n\n```python\n%pip install openai\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)\n\n\n\n\n\n\n\n    Collecting openai\n      Downloading openai-0.27.6-py3-none-any.whl (71 kB)\n    \u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m1."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 7a7d7160-1636-4a30-887e-08d518422e34
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7a7d7160-1636-4a30-887e-08d518422e34\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e7190cd5-797f-491c-95b9-dae4b65c71fd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"4c758903eaa6bea9a98c41dbe8f1376d023f0199aa6b3a0ceb6779a6d4510f98\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fe4349ab-f091-4038-8c3d-b6ef616838c0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9d8556a7c65a4f971d45585bf6ec0c76dbc138fe7cdd2bcc95d467074c37db4e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1852, \"end_char_idx\": 4267, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n\n```python\n%pip install openai\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)\n\n\n\n\n\n\n\n    Collecting openai\n      Downloading openai-0.27.6-py3-none-any.whl (71 kB)\n    \u001b[2K     \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n    \u001b[?25hRequirement already satisfied: tqdm in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (4.65.0)\n    Requirement already satisfied: requests>=2.20 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (2.28.2)\n    Requirement already satisfied: aiohttp in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (3.8.4)\n    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (1.26.14)\n    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)\n    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)\n    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2.1.1)\n    Requirement already satisfied: attrs>=17.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (22.2.0)\n    Requirement already satisfied: frozenlist>=1.1.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)\n    Requirement already satisfied: multidict<7.0,>=4.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)\n    Requirement already satisfied: yarl<2.0,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.8.2)\n    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (4.0."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: fe4349ab-f091-4038-8c3d-b6ef616838c0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fe4349ab-f091-4038-8c3d-b6ef616838c0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7a7d7160-1636-4a30-887e-08d518422e34\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"3615c9a61e90e315000ccc7dc54f53823a4c9d13d80f211695285849aea42513\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2b3fc3d6-c969-463f-8c34-49c4e9791983\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"13e62d288df42ea07bc9615c59b32325d0cc5c6e0da7004563f384634ae316c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2951, \"end_char_idx\": 5274, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)\n    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)\n    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2.1.1)\n    Requirement already satisfied: attrs>=17.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (22.2.0)\n    Requirement already satisfied: frozenlist>=1.1.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)\n    Requirement already satisfied: multidict<7.0,>=4.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)\n    Requirement already satisfied: yarl<2.0,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.8.2)\n    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (4.0.2)\n    Requirement already satisfied: aiosignal>=1.1.2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)\n    Installing collected packages: openai\n    Successfully installed openai-0.27.6\n    \n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip\u001b[0m\n    Note: you may need to restart the kernel to use updated packages.\n\n\n\n\n\n\n\n    Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n\n```python\n%pip install azure-kusto-data\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)\n\n\n\n\n\n\n\n    Requirement already satisfied: azure-kusto-data in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (4."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2b3fc3d6-c969-463f-8c34-49c4e9791983
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2b3fc3d6-c969-463f-8c34-49c4e9791983\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fe4349ab-f091-4038-8c3d-b6ef616838c0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"12f042a43aaf529f56d9d41b268f5e808b3890b14ee96288462312eb3ce801d1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dc5f0f09-1d9d-495f-b376-55c3477a2c1b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d263f5831585e4b0b36607b88507402a8388837623bc17963ce9d8db7da5a0de\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4222, \"end_char_idx\": 6483, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "10/site-packages (from aiohttp->openai) (4.0.2)\n    Requirement already satisfied: aiosignal>=1.1.2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)\n    Installing collected packages: openai\n    Successfully installed openai-0.27.6\n    \n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip\u001b[0m\n    Note: you may need to restart the kernel to use updated packages.\n\n\n\n\n\n\n\n    Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n\n```python\n%pip install azure-kusto-data\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)\n\n\n\n\n\n\n\n    Requirement already satisfied: azure-kusto-data in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (4.1.4)\n    Requirement already satisfied: msal<2,>=1.9.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.21.0)\n    Requirement already satisfied: python-dateutil>=2.8.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.8.2)\n    Requirement already satisfied: azure-core<2,>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.26.4)\n    Requirement already satisfied: requests>=2.13.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.28.2)\n    Requirement already satisfied: ijson~=3.1 in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (from azure-kusto-data) (3.2.0.post0)\n    Requirement already satisfied: azure-identity<2,>=1.5.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.12.0)\n    Requirement already satisfied: six>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (1.16."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: dc5f0f09-1d9d-495f-b376-55c3477a2c1b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"dc5f0f09-1d9d-495f-b376-55c3477a2c1b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2b3fc3d6-c969-463f-8c34-49c4e9791983\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"6a806adf8e85e8276f14c8725ef14d7350ed0a7285c01b3b2e6704219225f50d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1af47952-00b2-43cb-83bc-f562ce7f0a2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"43b6a42f3a930bf9a8feb2b0cf23aaa3b919c99fd96e082c8ab1861460c4bee2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5254, \"end_char_idx\": 7706, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "10/site-packages (4.1.4)\n    Requirement already satisfied: msal<2,>=1.9.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.21.0)\n    Requirement already satisfied: python-dateutil>=2.8.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.8.2)\n    Requirement already satisfied: azure-core<2,>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.26.4)\n    Requirement already satisfied: requests>=2.13.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.28.2)\n    Requirement already satisfied: ijson~=3.1 in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (from azure-kusto-data) (3.2.0.post0)\n    Requirement already satisfied: azure-identity<2,>=1.5.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.12.0)\n    Requirement already satisfied: six>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (1.16.0)\n    Requirement already satisfied: typing-extensions>=4.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (4.5.0)\n    Requirement already satisfied: cryptography>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (40.0.1)\n    Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (1.0.0)\n    Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal<2,>=1.9.0->azure-kusto-data) (2.6.0)\n    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (1.26.14)\n    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2.1.1)\n    Requirement already satisfied: idna<4,>=2."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 1af47952-00b2-43cb-83bc-f562ce7f0a2c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1af47952-00b2-43cb-83bc-f562ce7f0a2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dc5f0f09-1d9d-495f-b376-55c3477a2c1b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"ee13f766bd782735d293339d4fd671fb4819c91b9c2cc1b3ef5c25867f43f311\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"33224cf4-b00d-40f0-aef4-68d1122c6366\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"56feecf0ba36b28225d69ec59cffe2f9542b12e3538bcdc653311fb429261b63\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6449, \"end_char_idx\": 8800, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": ">=1.11.0->azure-kusto-data) (1.16.0)\n    Requirement already satisfied: typing-extensions>=4.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (4.5.0)\n    Requirement already satisfied: cryptography>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (40.0.1)\n    Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (1.0.0)\n    Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal<2,>=1.9.0->azure-kusto-data) (2.6.0)\n    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (1.26.14)\n    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2.1.1)\n    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (3.4)\n    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2022.12.7)\n    Requirement already satisfied: cffi>=1.12 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (1.15.1)\n    Requirement already satisfied: portalocker<3,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity<2,>=1.5.0->azure-kusto-data) (2.7.0)\n    Requirement already satisfied: pycparser in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (2.21)\n    \n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 33224cf4-b00d-40f0-aef4-68d1122c6366
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"33224cf4-b00d-40f0-aef4-68d1122c6366\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1af47952-00b2-43cb-83bc-f562ce7f0a2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"0552266ddeeebfec8cf67d40b6640fa97b5a9d5bb7791ae8c52d03dddcf04ead\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3c87a295-305b-4439-b53a-86f3a23428f7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6688e41007537baa45334a702bbc393a8dd0f916978bd1c220526aa8f9d3cc1f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7628, \"end_char_idx\": 9980, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "13.0->azure-kusto-data) (2.1.1)\n    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (3.4)\n    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2022.12.7)\n    Requirement already satisfied: cffi>=1.12 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (1.15.1)\n    Requirement already satisfied: portalocker<3,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity<2,>=1.5.0->azure-kusto-data) (2.7.0)\n    Requirement already satisfied: pycparser in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (2.21)\n    \n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip\u001b[0m\n    Note: you may need to restart the kernel to use updated packages.\n\n\n\n\n\n\n\n    Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n### Download precomputed Embeddings\n\n\n\nIn this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.\n\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)\n\n\n\n\n\n    'vector_database_wikipedia_articles_embedded.zip'\n\n\n\n\n```python\n\nimport zipfile\n\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"/lakehouse/default/Files/data\")\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 3c87a295-305b-4439-b53a-86f3a23428f7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3c87a295-305b-4439-b53a-86f3a23428f7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"33224cf4-b00d-40f0-aef4-68d1122c6366\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"f217d4500f5b19b9d46f813bc2986419fff61e289b2ab28538254d4ada43efe7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6a03e4a2-cab9-4917-8161-e09f8d96cf53\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2f58c2dc423f858fdd4c773f02d13745663c0f07b2cfe3584e1f75b04a2ec894\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8720, \"end_char_idx\": 10453, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n    \u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip\u001b[0m\n    Note: you may need to restart the kernel to use updated packages.\n\n\n\n\n\n\n\n    Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n### Download precomputed Embeddings\n\n\n\nIn this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.\n\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)\n\n\n\n\n\n    'vector_database_wikipedia_articles_embedded.zip'\n\n\n\n\n```python\n\nimport zipfile\n\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"/lakehouse/default/Files/data\")\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)\n\n\n\n```python\nimport pandas as pd\n\nfrom ast import literal_eval\n\narticle_df = pd.read_csv('/lakehouse/default/Files/data/vector_database_wikipedia_articles_embedded.csv')\n# Read vectors from strings back into a list\narticle_df[\"title_vector\"] = article_df.title_vector.apply(literal_eval)\narticle_df[\"content_vector\"] = article_df.content_vector.apply(literal_eval)\narticle_df.head()\n\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6a03e4a2-cab9-4917-8161-e09f8d96cf53
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6a03e4a2-cab9-4917-8161-e09f8d96cf53\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c87a295-305b-4439-b53a-86f3a23428f7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"b4bb1edffeaea08416745eb18034970b9b18dae51d4642de4f4e9c247b681de4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a0929a5f-c7f9-4b6e-a890-798572598620\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"420c1c7a1a4a1c15a3e8a9dd3de40c2324ff4c2b934628f0ef2e251689866033\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9087, \"end_char_idx\": 11470, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "Warning: PySpark kernel has been restarted to use updated packages.\n    \n\n\n### Download precomputed Embeddings\n\n\n\nIn this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.\n\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)\n\n\n\n\n\n    'vector_database_wikipedia_articles_embedded.zip'\n\n\n\n\n```python\n\nimport zipfile\n\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"/lakehouse/default/Files/data\")\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)\n\n\n\n```python\nimport pandas as pd\n\nfrom ast import literal_eval\n\narticle_df = pd.read_csv('/lakehouse/default/Files/data/vector_database_wikipedia_articles_embedded.csv')\n# Read vectors from strings back into a list\narticle_df[\"title_vector\"] = article_df.title_vector.apply(literal_eval)\narticle_df[\"content_vector\"] = article_df.content_vector.apply(literal_eval)\narticle_df.head()\n\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a0929a5f-c7f9-4b6e-a890-798572598620
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a0929a5f-c7f9-4b6e-a890-798572598620\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6a03e4a2-cab9-4917-8161-e09f8d96cf53\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"4566f47ef49daba7132989f45b43eba5d01ee7a7f4cee6e28db5b600c5cd30a5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1c2dc2ea-f26d-4296-b39d-67dfd85d09f3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c92f1909b1deb45242cd42f34e51d158a18331b00df6dd18f3d4dd75bb0332c0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10375, \"end_char_idx\": 12501, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.) is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 1c2dc2ea-f26d-4296-b39d-67dfd85d09f3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1c2dc2ea-f26d-4296-b39d-67dfd85d09f3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a0929a5f-c7f9-4b6e-a890-798572598620\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"2c33fd2cd0444361df088b0e3e72b7a1ae8c48c5b9376b88b824988ee374a0c1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7425717a-b88b-48b1-aeda-137fb1a860ca\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2229932697c0e43811fa611df27097cc2a7f81050bd63cb3a479f71573e44b8c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11471, \"end_char_idx\": 13727, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere. Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n### Store vectors in a Kusto table\n\n\nCreate a table & load the vectors in Kusto based on the contents in the dataframe. The spark option CreakeIfNotExists will automatically create a table if it doesn't exist\n\n\n\n```python\n# replace with your AAD Tenant ID, Kusto Cluster URI, Kusto DB name and Kusto Table\nAAD_TENANT_ID = \"\"\nKUSTO_CLUSTER =  \"\"\nKUSTO_DATABASE = \"Vector\"\nKUSTO_TABLE = \"Wiki\"\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 37, Finished, Available)\n\n\n\n```python\n\nkustoOptions = {\"kustoCluster\": KUSTO_CLUSTER, \"kustoDatabase\" :KUSTO_DATABASE, \"kustoTable\" : KUSTO_TABLE }\n\n# Replace the auth method based on your desired authentication mechanism  - https://github.com/Azure/azure-kusto-spark/blob/master/docs/Authentication.md\naccess_token=mssparkutils.credentials.getToken(kustoOptions[\"kustoCluster\"])\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 21, Finished, Available)\n\n\n\n```python\n#Pandas data frame to spark dataframe\nsparkDF=spark.createDataFrame(article_df)\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 7425717a-b88b-48b1-aeda-137fb1a860ca
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7425717a-b88b-48b1-aeda-137fb1a860ca\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1c2dc2ea-f26d-4296-b39d-67dfd85d09f3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c376eb1188813532edf2b3a1a472c4135f48e126ddd6e747d3d96745406a9763\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b4276cb4-9103-4c89-93d0-01111d5fb0c6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"948ef270714793d1c74fe9a89b7cac6865d0d6bc8d3da6ab7e6bda99903bc79c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12502, \"end_char_idx\": 15096, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n### Store vectors in a Kusto table\n\n\nCreate a table & load the vectors in Kusto based on the contents in the dataframe. The spark option CreakeIfNotExists will automatically create a table if it doesn't exist\n\n\n\n```python\n# replace with your AAD Tenant ID, Kusto Cluster URI, Kusto DB name and Kusto Table\nAAD_TENANT_ID = \"\"\nKUSTO_CLUSTER =  \"\"\nKUSTO_DATABASE = \"Vector\"\nKUSTO_TABLE = \"Wiki\"\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 37, Finished, Available)\n\n\n\n```python\n\nkustoOptions = {\"kustoCluster\": KUSTO_CLUSTER, \"kustoDatabase\" :KUSTO_DATABASE, \"kustoTable\" : KUSTO_TABLE }\n\n# Replace the auth method based on your desired authentication mechanism  - https://github.com/Azure/azure-kusto-spark/blob/master/docs/Authentication.md\naccess_token=mssparkutils.credentials.getToken(kustoOptions[\"kustoCluster\"])\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 21, Finished, Available)\n\n\n\n```python\n#Pandas data frame to spark dataframe\nsparkDF=spark.createDataFrame(article_df)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 22, Finished, Available)\n\n\n    /opt/spark/python/lib/pyspark.zip/pyspark/sql/pandas/conversion.py:604: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n\n\n\n```python\n# Write data to a Kusto table\nsparkDF.write. \\\nformat(\"com.microsoft.kusto.spark.synapse.datasource\"). \\\noption(\"kustoCluster\",kustoOptions[\"kustoCluster\"]). \\\noption(\"kustoDatabase\",kustoOptions[\"kustoDatabase\"]). \\\noption(\"kustoTable\", kustoOptions[\"kustoTable\"]). \\\noption(\"accessToken\", access_token). \\\noption(\"tableCreateOptions\", \"CreateIfNotExist\").\\\nmode(\"Append\"). \\\nsave()\n\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 23, Finished, Available)\n\n\n### Prepare your OpenAI API key\n# \n\nThe OpenAI API key is used for vectorization of the documents and queries. You can follow the instructions to create and retrieve your Azure OpenAI key and endpoint. https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings\n\n\nPlease make sure to use the `text-embedding-3-small` model. Since the precomputed embeddings were created with `text-embedding-3-small` model we also have to use it during search.\n\n\n\n```python\nimport openai\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)\n\n\n#### If using Azure Open AI"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: b4276cb4-9103-4c89-93d0-01111d5fb0c6
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"b4276cb4-9103-4c89-93d0-01111d5fb0c6\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7425717a-b88b-48b1-aeda-137fb1a860ca\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"9a37c87218504a3ac0943e9feeddef3524bf90196674b69520a939e7955db953\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"93742137-206c-439e-a169-90cf65882097\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c9e8e5be9bcf5fb8b27da96d341ecd7741ceaa63c9f0b91a92423093ebafaafc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13734, \"end_char_idx\": 16225, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 22, Finished, Available)\n\n\n    /opt/spark/python/lib/pyspark.zip/pyspark/sql/pandas/conversion.py:604: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n\n\n\n```python\n# Write data to a Kusto table\nsparkDF.write. \\\nformat(\"com.microsoft.kusto.spark.synapse.datasource\"). \\\noption(\"kustoCluster\",kustoOptions[\"kustoCluster\"]). \\\noption(\"kustoDatabase\",kustoOptions[\"kustoDatabase\"]). \\\noption(\"kustoTable\", kustoOptions[\"kustoTable\"]). \\\noption(\"accessToken\", access_token). \\\noption(\"tableCreateOptions\", \"CreateIfNotExist\").\\\nmode(\"Append\"). \\\nsave()\n\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 23, Finished, Available)\n\n\n### Prepare your OpenAI API key\n# \n\nThe OpenAI API key is used for vectorization of the documents and queries. You can follow the instructions to create and retrieve your Azure OpenAI key and endpoint. https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings\n\n\nPlease make sure to use the `text-embedding-3-small` model. Since the precomputed embeddings were created with `text-embedding-3-small` model we also have to use it during search.\n\n\n\n```python\nimport openai\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)\n\n\n#### If using Azure Open AI\n\n\n```python\nopenai.api_version = '2022-12-01'\nopenai.api_base = '' # Please add your endpoint here\nopenai.api_type = 'azure'\nopenai.api_key = ''  # Please add your api key here\n\ndef embed(query):\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n            input=query,\n            deployment_id=\"embed\", #replace with your deployment id\n            chunk_size=1\n    )[\"data\"][0][\"embedding\"]\n    return embedded_query\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 44, Finished, Available)\n\n\n#### If using Open AI\n\nOnly run this cell if you plan to use Open AI for embedding\n\n\n```python\nopenai.api_key = \"\"\n\n\ndef embed(query):\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n        input=query,\n        model=\"text-embedding-3-small\",\n    )[\"data\"][0][\"embedding\"]\n    return embedded_query\n```\n\n### Generate embedding for the search term\n\n\n```python\n\nsearchedEmbedding = embed(\"places where you worship\")\n#print(searchedEmbedding)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 45, Finished, Available)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 93742137-206c-439e-a169-90cf65882097
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"93742137-206c-439e-a169-90cf65882097\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b4276cb4-9103-4c89-93d0-01111d5fb0c6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"43acb5c7ecd5f2a3064f6fc222271c4d0d826d75d1bf014bf9740a28f927f85f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c60e01ef-9ef3-4476-a729-8ddc7bf2426c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f22cfe0ac1f9f23c98bf31f5fe975b784d2af3f489297996a9332ccd171d3f8c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14954, \"end_char_idx\": 17577, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "```python\nimport openai\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)\n\n\n#### If using Azure Open AI\n\n\n```python\nopenai.api_version = '2022-12-01'\nopenai.api_base = '' # Please add your endpoint here\nopenai.api_type = 'azure'\nopenai.api_key = ''  # Please add your api key here\n\ndef embed(query):\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n            input=query,\n            deployment_id=\"embed\", #replace with your deployment id\n            chunk_size=1\n    )[\"data\"][0][\"embedding\"]\n    return embedded_query\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 44, Finished, Available)\n\n\n#### If using Open AI\n\nOnly run this cell if you plan to use Open AI for embedding\n\n\n```python\nopenai.api_key = \"\"\n\n\ndef embed(query):\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n        input=query,\n        model=\"text-embedding-3-small\",\n    )[\"data\"][0][\"embedding\"]\n    return embedded_query\n```\n\n### Generate embedding for the search term\n\n\n```python\n\nsearchedEmbedding = embed(\"places where you worship\")\n#print(searchedEmbedding)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 45, Finished, Available)\n\n\n#### Semantic search in Kusto \n\nWe will search the Kusto table for the closest vectors.\n\nWe will be using the series-cosine-similarity-fl UDF for similarity search. \n\nPlease create the function in your database before proceeding -\nhttps://learn.microsoft.com/en-us/azure/data-explorer/kusto/functions-library/series-cosine-similarity-fl?tabs=query-defined\n\n\n```python\nfrom azure.kusto.data import KustoClient, KustoConnectionStringBuilder\nfrom azure.kusto.data.exceptions import KustoServiceError\nfrom azure.kusto.data.helpers import dataframe_from_result_table\nimport pandas as pd\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)\n\n\n\n```python\nKCSB = KustoConnectionStringBuilder.with_aad_device_authentication(\n    KUSTO_CLUSTER)\nKCSB.authority_id = AAD_TENANT_ID\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)\n\n\n\n```python\nKUSTO_CLIENT = KustoClient(KCSB)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)\n\n\n\n```python\nKUSTO_QUERY = \"Wiki | extend similarity = series_cosine_similarity_fl(dynamic(\"+str(searchedEmbedding)+\"), content_vector,1,1) | top 10 by similarity desc \"\n\nRESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c60e01ef-9ef3-4476-a729-8ddc7bf2426c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c60e01ef-9ef3-4476-a729-8ddc7bf2426c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"93742137-206c-439e-a169-90cf65882097\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"aace5e91cfd8d28e425f432acfd93ab13834e83049c87cc0352b895bb85eea19\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fd81df26-35c5-428a-9d68-8a9f2a72f4df\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f1b7445c92894888646041223e0b0ffcb3bb906c10bcbf4c255d6c4e61e0093c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16586, \"end_char_idx\": 17744, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "```python\nfrom azure.kusto.data import KustoClient, KustoConnectionStringBuilder\nfrom azure.kusto.data.exceptions import KustoServiceError\nfrom azure.kusto.data.helpers import dataframe_from_result_table\nimport pandas as pd\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)\n\n\n\n```python\nKCSB = KustoConnectionStringBuilder.with_aad_device_authentication(\n    KUSTO_CLUSTER)\nKCSB.authority_id = AAD_TENANT_ID\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)\n\n\n\n```python\nKUSTO_CLIENT = KustoClient(KCSB)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)\n\n\n\n```python\nKUSTO_QUERY = \"Wiki | extend similarity = series_cosine_similarity_fl(dynamic(\"+str(searchedEmbedding)+\"), content_vector,1,1) | top 10 by similarity desc \"\n\nRESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)\n\n\n\n```python\ndf = dataframe_from_result_table(RESPONSE.primary_results[0])\ndf\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 49, Finished, Available)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: fd81df26-35c5-428a-9d68-8a9f2a72f4df
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fd81df26-35c5-428a-9d68-8a9f2a72f4df\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c60e01ef-9ef3-4476-a729-8ddc7bf2426c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"48a09ac6c0f67289dedff65f26fbf56b0d1692301a0ed5be360cf0b1b136fbeb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5107ae41-7a66-4d8e-8a7f-9a99a9158cd2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"fdde13f20a2be24c2f3d0e8064fbfbef7b1d4f4369356c3db5e3a7d40642c503\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16586, \"end_char_idx\": 19642, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "```python\nfrom azure.kusto.data import KustoClient, KustoConnectionStringBuilder\nfrom azure.kusto.data.exceptions import KustoServiceError\nfrom azure.kusto.data.helpers import dataframe_from_result_table\nimport pandas as pd\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)\n\n\n\n```python\nKCSB = KustoConnectionStringBuilder.with_aad_device_authentication(\n    KUSTO_CLUSTER)\nKCSB.authority_id = AAD_TENANT_ID\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)\n\n\n\n```python\nKUSTO_CLIENT = KustoClient(KCSB)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)\n\n\n\n```python\nKUSTO_QUERY = \"Wiki | extend similarity = series_cosine_similarity_fl(dynamic(\"+str(searchedEmbedding)+\"), content_vector,1,1) | top 10 by similarity desc \"\n\nRESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)\n\n\n\n```python\ndf = dataframe_from_result_table(RESPONSE.primary_results[0])\ndf\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 49, Finished, Available)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n      <th>similarity</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>852</td>\n      <td>https://simple.wikipedia.org/wiki/Temple</td>\n      <td>Temple</td>\n      <td>A temple is a building where people go to prac...</td>\n      <td>[-0.021837441250681877, -0.007722342386841774,...</td>\n      <td>[-0.0019541378132998943, 0.007151313126087189,...</td>\n      <td>413</td>\n      <td>0.834495</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>78094</td>\n      <td>https://simple.wikipedia.org/wiki/Christian%20...</td>\n      <td>Christian worship</td>\n      <td>In Christianity, worship has been thought as b...</td>\n      <td>[0.0017675267299637198, -0.008890199474990368,...</td>\n      <td>[0.020530683919787407, 0.0024345638230443, -0....</td>\n      <td>20320</td>\n      <td>0.832132</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>59154</td>\n      <td>https://simple.wikipedia.org/wiki/Service%20of...</td>\n      <td>Service of worship</td>\n      <td>A service of worship is a religious meeting wh...</td>\n      <td>[-0.007969820871949196, 0.0004240311391185969,...</td>\n      <td>[0.003784010885283351, -0.0030924836173653603,...</td>\n      <td>15519</td>\n      <td>0.831633</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>51910</td>\n      <td>https://simple.wikipedia.org/wiki/Worship</td>\n      <td>Worship</td>\n      <td>Worship is a word often used in religion."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 5107ae41-7a66-4d8e-8a7f-9a99a9158cd2
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5107ae41-7a66-4d8e-8a7f-9a99a9158cd2\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fd81df26-35c5-428a-9d68-8a9f2a72f4df\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"9b3720c8915bb4821c30b52e5b43b54cf88350aa4d9a15f4db190eba560e5031\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"265b75d6-bbd4-4e01-9869-0ac2af87db26\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4049fe6e800733c814f0f34a9039f72c9bd1b4f4dd37eeeeec45e0e6afb661fb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19644, \"end_char_idx\": 21533, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "It ...</td>\n      <td>[0.0036036288365721703, -0.01276545226573944, ...</td>\n      <td>[0.007925753481686115, -0.0110504487529397, 0....</td>\n      <td>14010</td>\n      <td>0.828185</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>29576</td>\n      <td>https://simple.wikipedia.org/wiki/Altar</td>\n      <td>Altar</td>\n      <td>An altar is a place, often a table, where a re...</td>\n      <td>[0.007887467741966248, -0.02706138789653778, -...</td>\n      <td>[0.023901859298348427, -0.031175222247838977, ...</td>\n      <td>8708</td>\n      <td>0.824124</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>92507</td>\n      <td>https://simple.wikipedia.org/wiki/Shrine</td>\n      <td>Shrine</td>\n      <td>A shrine is a holy or sacred place with someth...</td>\n      <td>[-0.011601685546338558, 0.006366696208715439, ...</td>\n      <td>[0.016423320397734642, -0.0015560361789539456,...</td>\n      <td>23945</td>\n      <td>0.823863</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>815</td>\n      <td>https://simple.wikipedia.org/wiki/Synagogue</td>\n      <td>Synagogue</td>\n      <td>A synagogue is a place where Jews meet to wors...</td>\n      <td>[-0.017317570745944977, 0.0022673190105706453,...</td>\n      <td>[-0.004515442531555891, 0.003739549545571208, ...</td>\n      <td>398</td>\n      <td>0.819942</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>68080</td>\n      <td>https://simple.wikipedia.org/wiki/Shinto%20shrine</td>\n      <td>Shinto shrine</td>\n      <td>A Shinto shrine is a sacred place or site wher...</td>\n      <td>[0.0035740730818361044, 0.0028098472394049168,...</td>\n      <td>[0.011014971882104874, 0.00042272370774298906,...</td>\n      <td>18106</td>\n      <td>0.818475</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>57790</td>\n      <td>https://simple.wikipedia.org/wiki/Chapel</td>\n      <td>Chapel</td>\n      <td>A chapel is a place for Christian worship."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 265b75d6-bbd4-4e01-9869-0ac2af87db26
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"265b75d6-bbd4-4e01-9869-0ac2af87db26\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5107ae41-7a66-4d8e-8a7f-9a99a9158cd2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"70ef3f2c9bea7fa5ed8f5c3ee101ef9a52ea0bfabf354c096eb9c106b8aa0b56\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"46604e33-cc46-42b2-af1c-c720086f2a26\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e0933c6dfc9392ec545bb692e8b53e96ff82a5d58ee7b62053e6f39937bf838b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "The...</td>\n      <td>[-0.01371884811669588, 0.0031672674231231213, ...</td>\n      <td>[0.002526090247556567, 0.02482965588569641, 0....</td>\n      <td>15260</td>\n      <td>0.817608</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>142</td>\n      <td>https://simple.wikipedia.org/wiki/Church%20%28...</td>\n      <td>Church (building)</td>\n      <td>A church is a building that was constructed to...</td>\n      <td>[0.0021336888894438744, 0.0029748091474175453,...</td>\n      <td>[0.016109377145767212, 0.022908871993422508, 0...</td>\n      <td>74</td>\n      <td>0.812636</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\nsearchedEmbedding = embed(\"unfortunate events in history\")\n\n```\n\n\n```python\nKUSTO_QUERY = \"Wiki | extend similarity = series_cosine_similarity_fl(dynamic(\"+str(searchedEmbedding)+\"), title_vector,1,1) | top 10 by similarity desc \"\nRESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)\n\ndf = dataframe_from_result_table(RESPONSE.primary_results[0])\ndf\n```\n\n\n    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 52, Finished, Available)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n      <th>similarity</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>848</td>\n      <td>https://simple.wikipedia.org/wiki/Tragedy</td>\n      <td>Tragedy</td>\n      <td>In theatre, a tragedy as defined by Aristotle .</td>\n      <td>[-0.019502468407154083, -0.010160734876990318,</td>\n      <td>[-0.012951433658599854, -0.018836138769984245,</td>\n      <td>410</td>\n      <td>0.851848</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4469</td>\n      <td>https://simple.wikipedia.org/wiki/The%20Holocaust</td>\n      <td>The Holocaust</td>\n      <td>The Holocaust, sometimes called The Shoah (), ."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 46604e33-cc46-42b2-af1c-c720086f2a26
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"46604e33-cc46-42b2-af1c-c720086f2a26\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"265b75d6-bbd4-4e01-9869-0ac2af87db26\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"25a1d0559e13c018b2817a4644b9e46a8999f00f525f4428c4b27abf8866beeb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"303180bc-cbc4-4cae-b254-94fdf06150ee\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"417340c4907f781f8894eb80f27dc594cb70dddf5021d72057f71716d3cb1c21\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n      <th>similarity</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>848</td>\n      <td>https://simple.wikipedia.org/wiki/Tragedy</td>\n      <td>Tragedy</td>\n      <td>In theatre, a tragedy as defined by Aristotle .</td>\n      <td>[-0.019502468407154083, -0.010160734876990318,</td>\n      <td>[-0.012951433658599854, -0.018836138769984245,</td>\n      <td>410</td>\n      <td>0.851848</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4469</td>\n      <td>https://simple.wikipedia.org/wiki/The%20Holocaust</td>\n      <td>The Holocaust</td>\n      <td>The Holocaust, sometimes called The Shoah (), .</td>\n      <td>[-0.030233195051550865, -0.024401605129241943,</td>\n      <td>[-0.016398731619119644, -0.013267949223518372,</td>\n      <td>1203</td>\n      <td>0.847222</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>64216</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of historical plagues</td>\n      <td>This list contains famous or well documented o.</td>\n      <td>[-0.010667890310287476, -0.0003575817099772393.</td>\n      <td>[-0.010863155126571655, -0.0012196656316518784.</td>\n      <td>16859</td>\n      <td>0.844411</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4397</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of disasters</td>\n      <td>This is a list of disasters, both natural and .</td>\n      <td>[-0.02713736332952976, -0.005278210621327162, .</td>\n      <td>[-0.023679986596107483, -0.006126823835074902,</td>\n      <td>1158</td>\n      <td>0."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 303180bc-cbc4-4cae-b254-94fdf06150ee
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"303180bc-cbc4-4cae-b254-94fdf06150ee\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"46604e33-cc46-42b2-af1c-c720086f2a26\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"266fdc99df3642652eb29cc9fe3496b6d02db5feb666aaedf6271bdab9970031\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dafce2c2-734a-4e2c-8085-437e805076fc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"77c1b8171812972be76d3182c6486c2f84d822a4791d3aa429aa7714cd3bf9c1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "sometimes called The Shoah (), .</td>\n      <td>[-0.030233195051550865, -0.024401605129241943,</td>\n      <td>[-0.016398731619119644, -0.013267949223518372,</td>\n      <td>1203</td>\n      <td>0.847222</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>64216</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of historical plagues</td>\n      <td>This list contains famous or well documented o.</td>\n      <td>[-0.010667890310287476, -0.0003575817099772393.</td>\n      <td>[-0.010863155126571655, -0.0012196656316518784.</td>\n      <td>16859</td>\n      <td>0.844411</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>4397</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of disasters</td>\n      <td>This is a list of disasters, both natural and .</td>\n      <td>[-0.02713736332952976, -0.005278210621327162, .</td>\n      <td>[-0.023679986596107483, -0.006126823835074902,</td>\n      <td>1158</td>\n      <td>0.843063</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>23073</td>\n      <td>https://simple.wikipedia.org/wiki/Disaster</td>\n      <td>Disaster</td>\n      <td>A disaster is something very not good that hap.</td>\n      <td>[-0.018235962837934497, -0.020034968852996823,</td>\n      <td>[-0.02504003793001175, 0.007415903266519308, 0.</td>\n      <td>7251</td>\n      <td>0.840334</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>4382</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of terrorist incidents</td>\n      <td>The following is a list by date of acts and fa.</td>\n      <td>[-0.03989032283425331, -0.012808636762201786, .</td>\n      <td>[-0.045838188380002975, -0.01682935282588005, .</td>\n      <td>1149</td>\n      <td>0.836162</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>13528</td>\n      <td>https://simple.wikipedia.org/wiki/A%20Series%2.</td>\n      <td>A Series of Unfortunate Events</td>\n      <td>A Series of Unfortunate Events is a series of .</td>\n      <td>[0."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: dafce2c2-734a-4e2c-8085-437e805076fc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"dafce2c2-734a-4e2c-8085-437e805076fc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"303180bc-cbc4-4cae-b254-94fdf06150ee\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"94448199f2b955cfaf4caafa5739633663f35899e42d8755f0b92051e0e5b887\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9aedd650-448c-4643-a1f5-b512bd2007fa\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"21d1b12e6b36bf914c92e15aa2b66237ef51bd2a79c391324eab94432268a216\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "843063</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>23073</td>\n      <td>https://simple.wikipedia.org/wiki/Disaster</td>\n      <td>Disaster</td>\n      <td>A disaster is something very not good that hap.</td>\n      <td>[-0.018235962837934497, -0.020034968852996823,</td>\n      <td>[-0.02504003793001175, 0.007415903266519308, 0.</td>\n      <td>7251</td>\n      <td>0.840334</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>4382</td>\n      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>\n      <td>List of terrorist incidents</td>\n      <td>The following is a list by date of acts and fa.</td>\n      <td>[-0.03989032283425331, -0.012808636762201786, .</td>\n      <td>[-0.045838188380002975, -0.01682935282588005, .</td>\n      <td>1149</td>\n      <td>0.836162</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>13528</td>\n      <td>https://simple.wikipedia.org/wiki/A%20Series%2.</td>\n      <td>A Series of Unfortunate Events</td>\n      <td>A Series of Unfortunate Events is a series of .</td>\n      <td>[0.0010618815431371331, -0.0267023965716362, -.</td>\n      <td>[0.002801976166665554, -0.02904471382498741, -.</td>\n      <td>4347</td>\n      <td>0.835172</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>42874</td>\n      <td>https://simple.wikipedia.org/wiki/History%20of.</td>\n      <td>History of the world</td>\n      <td>The history of the world (also called human hi.</td>\n      <td>[0.0026915925554931164, -0.022206028923392296,</td>\n      <td>[0.013645033352077007, -0.005165994167327881, .</td>\n      <td>11672</td>\n      <td>0.830243</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>4452</td>\n      <td>https://simple.wikipedia.org/wiki/Accident</td>\n      <td>Accident</td>\n      <td>An accident is when something goes wrong when .</td>\n      <td>[-0.004075294826179743, -0.0059883203357458115.</td>\n      <td>[0.00926120299845934, 0.013705797493457794, 0.</td>\n      <td>1190</td>\n      <td>0."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 9aedd650-448c-4643-a1f5-b512bd2007fa
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"9aedd650-448c-4643-a1f5-b512bd2007fa\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"47c8b8a5-7d46-497f-bfd4-768aa7f0ea49\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dafce2c2-734a-4e2c-8085-437e805076fc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb\", \"title\": \"Kusto as a Vector database for AI embeddings\", \"tokens\": 8919, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"f62eaab12057c6165e8d4f6c4fad0405fd149a107422469c0fb933a838e2166f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "ref_doc_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "chroma:document": "</td>\n      <td>A Series of Unfortunate Events</td>\n      <td>A Series of Unfortunate Events is a series of .</td>\n      <td>[0.0010618815431371331, -0.0267023965716362, -.</td>\n      <td>[0.002801976166665554, -0.02904471382498741, -.</td>\n      <td>4347</td>\n      <td>0.835172</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>42874</td>\n      <td>https://simple.wikipedia.org/wiki/History%20of.</td>\n      <td>History of the world</td>\n      <td>The history of the world (also called human hi.</td>\n      <td>[0.0026915925554931164, -0.022206028923392296,</td>\n      <td>[0.013645033352077007, -0.005165994167327881, .</td>\n      <td>11672</td>\n      <td>0.830243</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>4452</td>\n      <td>https://simple.wikipedia.org/wiki/Accident</td>\n      <td>Accident</td>\n      <td>An accident is when something goes wrong when .</td>\n      <td>[-0.004075294826179743, -0.0059883203357458115.</td>\n      <td>[0.00926120299845934, 0.013705797493457794, 0.</td>\n      <td>1190</td>\n      <td>0.826898</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>324</td>\n      <td>https://simple.wikipedia.org/wiki/History</td>\n      <td>History</td>\n      <td>History is the study of past events. People kn...</td>\n      <td>[0.006603690329939127, -0.011856242083013058, ...</td>\n      <td>[0.0048830462619662285, 0.0032003086525946856,...</td>\n      <td>170</td>\n      <td>0.824645</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f4b28d5a-3314-414a-8851-30e87b61329d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/README.ipynb", "title": "Kusto as a Vector database", "tokens": 363, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f4b28d5a-3314-414a-8851-30e87b61329d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/README.ipynb\", \"title\": \"Kusto as a Vector database\", \"tokens\": 363, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6beb4da0-3ed7-4423-ae65-7abd82e3164b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/README.ipynb\", \"title\": \"Kusto as a Vector database\", \"tokens\": 363, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9ec102c8125dac468f4c00c162caee35feb43a3a6df9ee5811044be86f2e561a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1681, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6beb4da0-3ed7-4423-ae65-7abd82e3164b", "doc_id": "6beb4da0-3ed7-4423-ae65-7abd82e3164b", "ref_doc_id": "6beb4da0-3ed7-4423-ae65-7abd82e3164b", "chroma:document": "# Kusto as a Vector database\n\n\n\n[Azure Data Explorer aka Kusto](https://azure.microsoft.com/en-us/products/data-explorer) is a cloud-based data analytics service that enables users to perform advanced analytics on large datasets in real-time. It is particularly well-suited for handling large volumes of data, making it an excellent choice for storing and searching vectors.\n\nKusto supports a special data type called dynamic, which can store unstructured data such as arrays and properties bag. [Dynamic data type](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/scalar-data-types/dynamic) is perfect for storing vector values. You can further augment the vector value by storing metadata related to the original object as separate columns in your table.  \nKusto also supports in-built function [series_cosine_similarity_fl](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/functions-library/series-cosine-similarity-fl) to perform vector similarity searches.\n\n[Get started](https://aka.ms/kustofree) with Kusto for free. \n\n![Kusto_Vector](./images/kusto_vector_db.png)\n\n\n\n## Getting started with Kusto and Open AI embedding\n\n### Demo Scenario\n\n![Wiki_embeddings](./images/wiki_embeddings.png)\n\n![semantic_search_flow](./images/semantic_search_user_flow.png)\n\nIf you\u2019d like to try this demo, please follow the instructions in the [Notebook](Getting_started_with_kusto_and_openai_embeddings.ipynb).\n\nIt will allow you to -  \n\n1. Use precomputed embeddings created by OpenAI API. \n\n2. Store the embeddings in Kusto. \n\n3. Convert raw text query to an embedding with OpenAI API. \n\n4. Use Kusto to perform cosine similarity search in the stored embeddings."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 141c2bf0-45f4-4e40-97fd-a6c73bf13e5d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"141c2bf0-45f4-4e40-97fd-a6c73bf13e5d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"117045dd-8d4b-46cd-87ab-a6a0d3a086f0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2236303d6b8edacf79adf5cde2a465109b1bc6566bf95b6a89f9bc36006f737f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3307, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "# Using Hologres as a vector database for OpenAI embeddings\n\nThis notebook guides you step by step on using Hologres as a vector database for OpenAI embeddings.\n\nThis notebook presents an end-to-end process of:\n1. Using precomputed embeddings created by OpenAI API.\n2. Storing the embeddings in a cloud instance of Hologres.\n3. Converting raw text query to an embedding with OpenAI API.\n4. Using Hologres to perform the nearest neighbour search in the created collection.\n5. Provide large language models with the search results as context in prompt engineering\n\n### What is Hologres\n\n[Hologres](https://www.alibabacloud.com/help/en/hologres/latest/what-is-hologres) is a unified real-time data warehousing service developed by Alibaba Cloud. You can use Hologres to write, update, process, and analyze large amounts of data in real time. Hologres supports standard SQL syntax, is compatible with PostgreSQL, and supports most PostgreSQL functions. Hologres supports online analytical processing (OLAP) and ad hoc analysis for up to petabytes of data, and provides high-concurrency and low-latency online data services. Hologres supports fine-grained isolation of multiple workloads and enterprise-level security capabilities. Hologres is deeply integrated with MaxCompute, Realtime Compute for Apache Flink, and DataWorks, and provides full-stack online and offline data warehousing solutions for enterprises.\n\nHologres provides vector database functionality by adopting [Proxima](https://www.alibabacloud.com/help/en/hologres/latest/vector-processing).\n\nProxima is a high-performance software library developed by Alibaba DAMO Academy. It allows you to search for the nearest neighbors of vectors. Proxima provides higher stability and performance than similar open source software such as Facebook AI Similarity Search (Faiss). Proxima provides basic modules that have leading performance and effects in the industry and allows you to search for similar images, videos, or human faces. Hologres is deeply integrated with Proxima to provide a high-performance vector search service.\n\n### Deployment options\n\n- [Click here](https://www.alibabacloud.com/product/hologres) to fast deploy [Hologres data warehouse](https://www.alibabacloud.com/help/en/hologres/latest/getting-started).\n\n\n## Prerequisites\n\nFor the purposes of this exercise we need to prepare a couple of things:\n\n1. Hologres cloud server instance.\n2. The 'psycopg2-binary' library to interact with the vector database. Any other postgresql client library is ok.\n3. An [OpenAI API key](https://beta.openai.com/account/api-keys).\n\n\n\nWe might validate if the server was launched successfully by running a simple curl command:\n\n\n### Install requirements\n\nThis notebook obviously requires the `openai` and `psycopg2-binary` packages, but there are also some other additional libraries we will use. The following command installs them all:\n\n\n\n```python\n! pip install openai psycopg2-binary pandas wget\n```\n\n### Prepare your OpenAI API key\n\nThe OpenAI API key is used for vectorization of the documents and queries.\n\nIf you don't have an OpenAI API key, you can get one from [https://beta.openai.com/account/api-keys](https://beta.openai.com/account/api-keys).\n\nOnce you get your key, please add it to your environment variables as `OPENAI_API_KEY`."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 117045dd-8d4b-46cd-87ab-a6a0d3a086f0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"117045dd-8d4b-46cd-87ab-a6a0d3a086f0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"141c2bf0-45f4-4e40-97fd-a6c73bf13e5d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"638add20a1592838a5a10086d7d548e51f22c5b15a3edcc7c948d4d37c17669e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f1496051-7ba9-445f-8fb3-d244fceb27b9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c4105f6c713872c0885ec57f4cd51584f75041811d3e2871f13eb9ba7cfc715e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2286, \"end_char_idx\": 5357, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "## Prerequisites\n\nFor the purposes of this exercise we need to prepare a couple of things:\n\n1. Hologres cloud server instance.\n2. The 'psycopg2-binary' library to interact with the vector database. Any other postgresql client library is ok.\n3. An [OpenAI API key](https://beta.openai.com/account/api-keys).\n\n\n\nWe might validate if the server was launched successfully by running a simple curl command:\n\n\n### Install requirements\n\nThis notebook obviously requires the `openai` and `psycopg2-binary` packages, but there are also some other additional libraries we will use. The following command installs them all:\n\n\n\n```python\n! pip install openai psycopg2-binary pandas wget\n```\n\n### Prepare your OpenAI API key\n\nThe OpenAI API key is used for vectorization of the documents and queries.\n\nIf you don't have an OpenAI API key, you can get one from [https://beta.openai.com/account/api-keys](https://beta.openai.com/account/api-keys).\n\nOnce you get your key, please add it to your environment variables as `OPENAI_API_KEY`.\n\n\n```python\n# Test that your OpenAI API key is correctly set as an environment variable\n# Note. if you run this notebook locally, you will need to reload your terminal and the notebook for the env variables to be live.\nimport os\n\n# Note. alternatively you can set a temporary env variable like this:\n# os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\"\n\nif os.getenv(\"OPENAI_API_KEY\") is not None:\n    print(\"OPENAI_API_KEY is ready\")\nelse:\n    print(\"OPENAI_API_KEY environment variable not found\")\n```\n\n    OPENAI_API_KEY is ready\n\n\n## Connect to Hologres\nFirst add it to your environment variables. or you can just change the \"psycopg2.connect\" parameters below\n\nConnecting to a running instance of Hologres server is easy with the official Python library:\n\n\n```python\nimport os\nimport psycopg2\n\n# Note. alternatively you can set a temporary env variable like this:\n# os.environ[\"PGHOST\"] = \"your_host\"\n# os.environ[\"PGPORT\"] \"5432\"),\n# os.environ[\"PGDATABASE\"] \"postgres\"),\n# os.environ[\"PGUSER\"] \"user\"),\n# os.environ[\"PGPASSWORD\"] \"password\"),\n\nconnection = psycopg2.connect(\n    host=os.environ.get(\"PGHOST\", \"localhost\"),\n    port=os.environ.get(\"PGPORT\", \"5432\"),\n    database=os.environ.get(\"PGDATABASE\", \"postgres\"),\n    user=os.environ.get(\"PGUSER\", \"user\"),\n    password=os.environ.get(\"PGPASSWORD\", \"password\")\n)\nconnection.set_session(autocommit=True)\n\n# Create a new cursor object\ncursor = connection.cursor()\n```\n\nWe can test the connection by running any available method:\n\n\n```python\n\n# Execute a simple query to test the connection\ncursor.execute(\"SELECT 1;\")\nresult = cursor.fetchone()\n\n# Check the query result\nif result == (1,):\n    print(\"Connection successful!\")\nelse:\n    print(\"Connection failed.\")\n```\n\n    Connection successful!\n\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\nThe downloaded file has to be then extracted:"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f1496051-7ba9-445f-8fb3-d244fceb27b9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f1496051-7ba9-445f-8fb3-d244fceb27b9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"117045dd-8d4b-46cd-87ab-a6a0d3a086f0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2625ee81c2020a1493840169f3cb2b709d0a13a4bd72d17e0cba7de92f5b52d7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1067453b-810c-49e6-bbe6-c1d856889c30\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"66d9df8f734b22fb4b9d9e2088f6f8c0736f964dd981da6a018711f2c379a2ae\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3869, \"end_char_idx\": 6823, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "## Connect to Hologres\nFirst add it to your environment variables. or you can just change the \"psycopg2.connect\" parameters below\n\nConnecting to a running instance of Hologres server is easy with the official Python library:\n\n\n```python\nimport os\nimport psycopg2\n\n# Note. alternatively you can set a temporary env variable like this:\n# os.environ[\"PGHOST\"] = \"your_host\"\n# os.environ[\"PGPORT\"] \"5432\"),\n# os.environ[\"PGDATABASE\"] \"postgres\"),\n# os.environ[\"PGUSER\"] \"user\"),\n# os.environ[\"PGPASSWORD\"] \"password\"),\n\nconnection = psycopg2.connect(\n    host=os.environ.get(\"PGHOST\", \"localhost\"),\n    port=os.environ.get(\"PGPORT\", \"5432\"),\n    database=os.environ.get(\"PGDATABASE\", \"postgres\"),\n    user=os.environ.get(\"PGUSER\", \"user\"),\n    password=os.environ.get(\"PGPASSWORD\", \"password\")\n)\nconnection.set_session(autocommit=True)\n\n# Create a new cursor object\ncursor = connection.cursor()\n```\n\nWe can test the connection by running any available method:\n\n\n```python\n\n# Execute a simple query to test the connection\ncursor.execute(\"SELECT 1;\")\nresult = cursor.fetchone()\n\n# Check the query result\nif result == (1,):\n    print(\"Connection successful!\")\nelse:\n    print(\"Connection failed.\")\n```\n\n    Connection successful!\n\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\nThe downloaded file has to be then extracted:\n\n\n```python\nimport zipfile\nimport os\nimport re\nimport tempfile\n\ncurrent_directory = os.getcwd()\nzip_file_path = os.path.join(current_directory, \"vector_database_wikipedia_articles_embedded.zip\")\noutput_directory = os.path.join(current_directory, \"../../data\")\n\nwith zipfile.ZipFile(zip_file_path, \"r\") as zip_ref:\n    zip_ref.extractall(output_directory)\n\n\n# check the csv file exist\nfile_name = \"vector_database_wikipedia_articles_embedded.csv\"\ndata_directory = os.path.join(current_directory, \"../../data\")\nfile_path = os.path.join(data_directory, file_name)\n\n\nif os.path.exists(file_path):\n    print(f\"The file {file_name} exists in the data directory.\")\nelse:\n    print(f\"The file {file_name} does not exist in the data directory.\")\n\n```\n\n    The file vector_database_wikipedia_articles_embedded.csv exists in the data directory.\n\n\n## Load data\n\nIn this section we are going to load the data prepared previous to this session, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.\n\n\n```python\n!unzip -n vector_database_wikipedia_articles_embedded.zip\n!ls -lh vector_database_wikipedia_articles_embedded.csv\n```\n\n    Archive:  vector_database_wikipedia_articles_embedded.zip\n    -rw-r--r--@ 1 geng  staff   1.7G Jan 31 01:19 vector_database_wikipedia_articles_embedded.csv\n\n\nTake a look at the data.\n\n\n```python\nimport pandas, json\ndata = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')\ndata\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 1067453b-810c-49e6-bbe6-c1d856889c30
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1067453b-810c-49e6-bbe6-c1d856889c30\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f1496051-7ba9-445f-8fb3-d244fceb27b9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"56ced9361d6401f6762c2b895c88b3e6929d2152ee6edc7caf79fb66348093fd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0ef99c20-21d7-4e02-a37b-d29ec8681a81\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dcb5fdf35242faba40e9d7263ad8f6db55a2fd6b0f845c50e155f89f47c218fe\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5095, \"end_char_idx\": 7839, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\nThe downloaded file has to be then extracted:\n\n\n```python\nimport zipfile\nimport os\nimport re\nimport tempfile\n\ncurrent_directory = os.getcwd()\nzip_file_path = os.path.join(current_directory, \"vector_database_wikipedia_articles_embedded.zip\")\noutput_directory = os.path.join(current_directory, \"../../data\")\n\nwith zipfile.ZipFile(zip_file_path, \"r\") as zip_ref:\n    zip_ref.extractall(output_directory)\n\n\n# check the csv file exist\nfile_name = \"vector_database_wikipedia_articles_embedded.csv\"\ndata_directory = os.path.join(current_directory, \"../../data\")\nfile_path = os.path.join(data_directory, file_name)\n\n\nif os.path.exists(file_path):\n    print(f\"The file {file_name} exists in the data directory.\")\nelse:\n    print(f\"The file {file_name} does not exist in the data directory.\")\n\n```\n\n    The file vector_database_wikipedia_articles_embedded.csv exists in the data directory.\n\n\n## Load data\n\nIn this section we are going to load the data prepared previous to this session, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.\n\n\n```python\n!unzip -n vector_database_wikipedia_articles_embedded.zip\n!ls -lh vector_database_wikipedia_articles_embedded.csv\n```\n\n    Archive:  vector_database_wikipedia_articles_embedded.zip\n    -rw-r--r--@ 1 geng  staff   1.7G Jan 31 01:19 vector_database_wikipedia_articles_embedded.csv\n\n\nTake a look at the data.\n\n\n```python\nimport pandas, json\ndata = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')\ndata\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0ef99c20-21d7-4e02-a37b-d29ec8681a81
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0ef99c20-21d7-4e02-a37b-d29ec8681a81\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1067453b-810c-49e6-bbe6-c1d856889c30\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c71d0254c86be16a95be8d4eaa32a68f2b4ccf27caf54a336d731fdc3c6f4d58\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a184ba02-d23c-48dc-9ef2-94f292cbdd40\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"011b0d32711c90c16319018a035728a3c0ca2a04dfb1efed79bd335d8a6f47c0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6673, \"end_char_idx\": 8870, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "Take a look at the data.\n\n\n```python\nimport pandas, json\ndata = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')\ndata\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.) is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a184ba02-d23c-48dc-9ef2-94f292cbdd40
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a184ba02-d23c-48dc-9ef2-94f292cbdd40\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0ef99c20-21d7-4e02-a37b-d29ec8681a81\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e817952481f58ba0aaead98384419120b6411207e9b77231f49273822ff6ce2c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dc134d6b-3fb3-4df1-bd78-81b39ed325ee\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"470d5828e8cd532d739e910589b771a421cb39c440084aff18376f3ca8bd37bc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere. Air is a.</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0.</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -.</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>.</th>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n    </tr>\n    <tr>\n      <th>24995</th>\n      <td>98295</td>\n      <td>https://simple.wikipedia.org/wiki/Geneva</td>\n      <td>Geneva</td>\n      <td>Geneva (,  ,  ,  , ) is the second biggest cit.</td>\n      <td>[-0.015773078426718712, 0.01737344264984131, 0.</td>\n      <td>[0.008000412955880165, 0.02008531428873539, 0.</td>\n      <td>24995</td>\n    </tr>\n    <tr>\n      <th>24996</th>\n      <td>98316</td>\n      <td>https://simple.wikipedia.org/wiki/Concubinage</td>\n      <td>Concubinage</td>\n      <td>Concubinage is the state of a woman in a relat.</td>\n      <td>[-0.00519518880173564,"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: dc134d6b-3fb3-4df1-bd78-81b39ed325ee
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"dc134d6b-3fb3-4df1-bd78-81b39ed325ee\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a184ba02-d23c-48dc-9ef2-94f292cbdd40\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"8046964a1921a60e9cdd03ea8ab33010c7555e2f9f5869909e7df0f2387120ba\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"206f11fb-58fa-4833-aef2-407df139491a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"831cab4bd3222fceba166e0d8e4406d5e7d8e5a5cd14fad71ed69eab3d0c360f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "Air is a.</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0.</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -.</td>\n      <td>4</td>\n    </tr>\n    <tr>\n      <th>.</th>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n    </tr>\n    <tr>\n      <th>24995</th>\n      <td>98295</td>\n      <td>https://simple.wikipedia.org/wiki/Geneva</td>\n      <td>Geneva</td>\n      <td>Geneva (,  ,  ,  , ) is the second biggest cit.</td>\n      <td>[-0.015773078426718712, 0.01737344264984131, 0.</td>\n      <td>[0.008000412955880165, 0.02008531428873539, 0.</td>\n      <td>24995</td>\n    </tr>\n    <tr>\n      <th>24996</th>\n      <td>98316</td>\n      <td>https://simple.wikipedia.org/wiki/Concubinage</td>\n      <td>Concubinage</td>\n      <td>Concubinage is the state of a woman in a relat.</td>\n      <td>[-0.00519518880173564, 0.005898841191083193, 0.</td>\n      <td>[-0.01736736111342907, -0.002740012714639306, .</td>\n      <td>24996</td>\n    </tr>\n    <tr>\n      <th>24997</th>\n      <td>98318</td>\n      <td>https://simple.wikipedia.org/wiki/Mistress%20%.</td>\n      <td>Mistress (lover)</td>\n      <td>A mistress is a man's long term female sexual .</td>\n      <td>[-0.023164259269833565, -0.02052430994808674, .</td>\n      <td>[-0.017878392711281776, -0.0004517830966506153.</td>\n      <td>24997</td>\n    </tr>\n    <tr>\n      <th>24998</th>\n      <td>98326</td>\n      <td>https://simple.wikipedia.org/wiki/Eastern%20Front</td>\n      <td>Eastern Front</td>\n      <td>Eastern Front can be one of the following:\\n\\n.</td>\n      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>\n      <td>[-0.0019235472427681088, -0.004023272544145584.</td>\n      <td>24998</td>\n    </tr>\n    <tr>\n      <th>24999</th>\n      <td>98327</td>\n      <td>https://simple.wikipedia.org/wiki/Italian%20Ca."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 206f11fb-58fa-4833-aef2-407df139491a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"206f11fb-58fa-4833-aef2-407df139491a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dc134d6b-3fb3-4df1-bd78-81b39ed325ee\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cca9a0e32d42e6f392e7542bb98702c4ae613ccbe6ed270dd4da796f40f77aa4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"afe153c1-7dc8-45ce-b858-83762cc8ccc5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"65f8d3530eee2759b889e843ca6fe6dc21293706f007647d80f094916678e9a9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "</td>\n      <td>[-0.00519518880173564, 0.005898841191083193, 0.</td>\n      <td>[-0.01736736111342907, -0.002740012714639306, .</td>\n      <td>24996</td>\n    </tr>\n    <tr>\n      <th>24997</th>\n      <td>98318</td>\n      <td>https://simple.wikipedia.org/wiki/Mistress%20%.</td>\n      <td>Mistress (lover)</td>\n      <td>A mistress is a man's long term female sexual .</td>\n      <td>[-0.023164259269833565, -0.02052430994808674, .</td>\n      <td>[-0.017878392711281776, -0.0004517830966506153.</td>\n      <td>24997</td>\n    </tr>\n    <tr>\n      <th>24998</th>\n      <td>98326</td>\n      <td>https://simple.wikipedia.org/wiki/Eastern%20Front</td>\n      <td>Eastern Front</td>\n      <td>Eastern Front can be one of the following:\\n\\n.</td>\n      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>\n      <td>[-0.0019235472427681088, -0.004023272544145584.</td>\n      <td>24998</td>\n    </tr>\n    <tr>\n      <th>24999</th>\n      <td>98327</td>\n      <td>https://simple.wikipedia.org/wiki/Italian%20Ca.</td>\n      <td>Italian Campaign</td>\n      <td>Italian Campaign can mean the following:\\n\\nTh.</td>\n      <td>[-0.014151256531476974, -0.008553029969334602,</td>\n      <td>[-0.011758845299482346, -0.01346028596162796, .</td>\n      <td>24999</td>\n    </tr>\n  </tbody>\n</table>\n<p>25000 rows \u00d7 7 columns</p>\n</div>\n\n\n\n\n```python\ntitle_vector_length = len(json.loads(data['title_vector'].iloc[0]))\ncontent_vector_length = len(json.loads(data['content_vector'].iloc[0]))\n\nprint(title_vector_length, content_vector_length)\n```\n\n    1536 1536\n\n\n### Create table and proxima vector index\n\nHologres stores data in __tables__ where each object is described by at least one vector. Our table will be called **articles** and each object will be described by both **title** and **content** vectors.\n\nWe will start with creating a table and create proxima indexes on both **title** and **content**, and then we will fill it with our precomputed embeddings."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: afe153c1-7dc8-45ce-b858-83762cc8ccc5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"afe153c1-7dc8-45ce-b858-83762cc8ccc5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"206f11fb-58fa-4833-aef2-407df139491a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"8136a08c6de5af07a28f349d74d6b6ed726a76d4821ebba3d613d14e9ab86843\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e12ae5f1-5701-496a-b6a6-bc3ed923aa9e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"da80087ba7bbac796d5e187be303cea52e7f10533f41f21c528b5e23684826eb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "</td>\n      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>\n      <td>[-0.0019235472427681088, -0.004023272544145584.</td>\n      <td>24998</td>\n    </tr>\n    <tr>\n      <th>24999</th>\n      <td>98327</td>\n      <td>https://simple.wikipedia.org/wiki/Italian%20Ca.</td>\n      <td>Italian Campaign</td>\n      <td>Italian Campaign can mean the following:\\n\\nTh.</td>\n      <td>[-0.014151256531476974, -0.008553029969334602,</td>\n      <td>[-0.011758845299482346, -0.01346028596162796, .</td>\n      <td>24999</td>\n    </tr>\n  </tbody>\n</table>\n<p>25000 rows \u00d7 7 columns</p>\n</div>\n\n\n\n\n```python\ntitle_vector_length = len(json.loads(data['title_vector'].iloc[0]))\ncontent_vector_length = len(json.loads(data['content_vector'].iloc[0]))\n\nprint(title_vector_length, content_vector_length)\n```\n\n    1536 1536\n\n\n### Create table and proxima vector index\n\nHologres stores data in __tables__ where each object is described by at least one vector. Our table will be called **articles** and each object will be described by both **title** and **content** vectors.\n\nWe will start with creating a table and create proxima indexes on both **title** and **content**, and then we will fill it with our precomputed embeddings.\n\n\n```python\ncursor.execute('CREATE EXTENSION IF NOT EXISTS proxima;')\ncreate_proxima_table_sql = '''\nBEGIN;\nDROP TABLE IF EXISTS articles;\nCREATE TABLE articles (\n    id INT PRIMARY KEY NOT NULL,\n    url TEXT,\n    title TEXT,\n    content TEXT,\n    title_vector float4[] check(\n        array_ndims(title_vector) = 1 and \n        array_length(title_vector, 1) = 1536\n    ), -- define the vectors\n    content_vector float4[] check(\n        array_ndims(content_vector) = 1 and \n        array_length(content_vector, 1) = 1536\n    ),\n    vector_id INT\n);\n\n-- Create indexes for the vector fields.\ncall set_table_property(\n    'articles',\n    'proxima_vectors', \n    '{\n        \"title_vector\":{\"algorithm\":\"Graph\",\"distance_method\":\"Euclidean\",\"builder_params\":{\"min_flush_proxima_row_count\" : 10}},\n        \"content_vector\":{\"algorithm\":\"Graph\",\"distance_method\":\"Euclidean\",\"builder_params\":{\"min_flush_proxima_row_count\" : 10}}\n    }'\n);  \n\nCOMMIT;\n'''\n\n# Execute the SQL statements (will autocommit)\ncursor.execute(create_proxima_table_sql)\n```\n\n### Upload data\n\nNow let's upload the data to the Hologres cloud instance using [COPY statement](https://www.alibabacloud.com/help/en/hologres/latest/use-the-copy-statement-to-import-or-export-data). This might take 5-10 minutes according to the network bandwidth."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e12ae5f1-5701-496a-b6a6-bc3ed923aa9e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e12ae5f1-5701-496a-b6a6-bc3ed923aa9e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"afe153c1-7dc8-45ce-b858-83762cc8ccc5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"704bc636bb0675ec790f79e2379c3d3a87bb98585baf0d81d8378bee30f97e93\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"20760963-9d2f-4096-950d-e1695519a9b9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5fcfaaf6c49a35c048fdc4153f39b4b43acec78e7fe85381fc11b94bc2da053f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11745, \"end_char_idx\": 14763, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "```python\ncursor.execute('CREATE EXTENSION IF NOT EXISTS proxima;')\ncreate_proxima_table_sql = '''\nBEGIN;\nDROP TABLE IF EXISTS articles;\nCREATE TABLE articles (\n    id INT PRIMARY KEY NOT NULL,\n    url TEXT,\n    title TEXT,\n    content TEXT,\n    title_vector float4[] check(\n        array_ndims(title_vector) = 1 and \n        array_length(title_vector, 1) = 1536\n    ), -- define the vectors\n    content_vector float4[] check(\n        array_ndims(content_vector) = 1 and \n        array_length(content_vector, 1) = 1536\n    ),\n    vector_id INT\n);\n\n-- Create indexes for the vector fields.\ncall set_table_property(\n    'articles',\n    'proxima_vectors', \n    '{\n        \"title_vector\":{\"algorithm\":\"Graph\",\"distance_method\":\"Euclidean\",\"builder_params\":{\"min_flush_proxima_row_count\" : 10}},\n        \"content_vector\":{\"algorithm\":\"Graph\",\"distance_method\":\"Euclidean\",\"builder_params\":{\"min_flush_proxima_row_count\" : 10}}\n    }'\n);  \n\nCOMMIT;\n'''\n\n# Execute the SQL statements (will autocommit)\ncursor.execute(create_proxima_table_sql)\n```\n\n### Upload data\n\nNow let's upload the data to the Hologres cloud instance using [COPY statement](https://www.alibabacloud.com/help/en/hologres/latest/use-the-copy-statement-to-import-or-export-data). This might take 5-10 minutes according to the network bandwidth.\n\n\n```python\nimport io\n\n# Path to the unzipped CSV file\ncsv_file_path = '../../data/vector_database_wikipedia_articles_embedded.csv'\n\n# In SQL, arrays are surrounded by {}, rather than []\ndef process_file(file_path):\n    with open(file_path, 'r') as file:\n        for line in file:\n            # Replace '[' with '{' and ']' with '}'\n            modified_line = line.replace('[', '{').replace(']', '}')\n            yield modified_line\n\n# Create a StringIO object to store the modified lines\nmodified_lines = io.StringIO(''.join(list(process_file(csv_file_path))))\n\n# Create the COPY command for the copy_expert method\ncopy_command = '''\nCOPY public.articles (id, url, title, content, title_vector, content_vector, vector_id)\nFROM STDIN WITH (FORMAT CSV, HEADER true, DELIMITER ',');\n'''\n\n# Execute the COPY command using the copy_expert method\ncursor.copy_expert(copy_command, modified_lines)\n```\n\nThe proxima index will be built in the background. We can do searching during this period but the query will be slow without the vector index. Use this command to wait for finish building the index.\n\n\n```python\ncursor.execute('vacuum articles;')\n```\n\n\n```python\n# Check the collection size to make sure all the points have been stored\ncount_sql = \"select count(*) from articles;\"\ncursor.execute(count_sql)\nresult = cursor.fetchone()\nprint(f\"Count:{result[0]}\")\n\n```\n\n    Count:25000\n\n\n## Search data\n\nOnce the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 20760963-9d2f-4096-950d-e1695519a9b9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"20760963-9d2f-4096-950d-e1695519a9b9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e12ae5f1-5701-496a-b6a6-bc3ed923aa9e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cf16b0e2c9b0bef2b2418a91f4b90bce4a8ba313525ca7183f45bd051e9f6793\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1b740b85-289c-418f-8cdc-f5215bac8f5d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"fce9e138c89316a953679eff10b09108029dd886ae9487c5920506bd7caab72a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13053, \"end_char_idx\": 15525, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "```python\nimport io\n\n# Path to the unzipped CSV file\ncsv_file_path = '../../data/vector_database_wikipedia_articles_embedded.csv'\n\n# In SQL, arrays are surrounded by {}, rather than []\ndef process_file(file_path):\n    with open(file_path, 'r') as file:\n        for line in file:\n            # Replace '[' with '{' and ']' with '}'\n            modified_line = line.replace('[', '{').replace(']', '}')\n            yield modified_line\n\n# Create a StringIO object to store the modified lines\nmodified_lines = io.StringIO(''.join(list(process_file(csv_file_path))))\n\n# Create the COPY command for the copy_expert method\ncopy_command = '''\nCOPY public.articles (id, url, title, content, title_vector, content_vector, vector_id)\nFROM STDIN WITH (FORMAT CSV, HEADER true, DELIMITER ',');\n'''\n\n# Execute the COPY command using the copy_expert method\ncursor.copy_expert(copy_command, modified_lines)\n```\n\nThe proxima index will be built in the background. We can do searching during this period but the query will be slow without the vector index. Use this command to wait for finish building the index.\n\n\n```python\ncursor.execute('vacuum articles;')\n```\n\n\n```python\n# Check the collection size to make sure all the points have been stored\ncount_sql = \"select count(*) from articles;\"\ncursor.execute(count_sql)\nresult = cursor.fetchone()\nprint(f\"Count:{result[0]}\")\n\n```\n\n    Count:25000\n\n\n## Search data\n\nOnce the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.\n\n\n\n```python\nimport openai\ndef query_knn(query, table_name, vector_name=\"title_vector\", top_k=20):\n\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n        input=query,\n        model=\"text-embedding-3-small\",\n    )[\"data\"][0][\"embedding\"]\n\n    # Convert the embedded_query to PostgreSQL compatible format\n    embedded_query_pg = \"{\" + \",\".join(map(str, embedded_query)) + \"}\"\n\n    # Create SQL query\n    query_sql = f\"\"\"\n    SELECT id, url, title, pm_approx_euclidean_distance({vector_name},'{embedded_query_pg}'::float4[]) AS distance\n    FROM {table_name}\n    ORDER BY distance\n    LIMIT {top_k};\n    \"\"\"\n    # Execute the query\n    cursor.execute(query_sql)\n    results = cursor.fetchall()\n\n    return results\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 1b740b85-289c-418f-8cdc-f5215bac8f5d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1b740b85-289c-418f-8cdc-f5215bac8f5d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"20760963-9d2f-4096-950d-e1695519a9b9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"50b95cd9d07101d70376392814db3ec6481dbea47789abb9c39b02024ee9be1f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a21902d1-a625-4700-a2c4-4d512be6953e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"87ae9784febfe831317d44fdaf3b2e3b385bae5e5cb7b94ebe92c364ede354e0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14149, \"end_char_idx\": 16421, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "```python\ncursor.execute('vacuum articles;')\n```\n\n\n```python\n# Check the collection size to make sure all the points have been stored\ncount_sql = \"select count(*) from articles;\"\ncursor.execute(count_sql)\nresult = cursor.fetchone()\nprint(f\"Count:{result[0]}\")\n\n```\n\n    Count:25000\n\n\n## Search data\n\nOnce the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.\n\n\n\n```python\nimport openai\ndef query_knn(query, table_name, vector_name=\"title_vector\", top_k=20):\n\n    # Creates embedding vector from user query\n    embedded_query = openai.Embedding.create(\n        input=query,\n        model=\"text-embedding-3-small\",\n    )[\"data\"][0][\"embedding\"]\n\n    # Convert the embedded_query to PostgreSQL compatible format\n    embedded_query_pg = \"{\" + \",\".join(map(str, embedded_query)) + \"}\"\n\n    # Create SQL query\n    query_sql = f\"\"\"\n    SELECT id, url, title, pm_approx_euclidean_distance({vector_name},'{embedded_query_pg}'::float4[]) AS distance\n    FROM {table_name}\n    ORDER BY distance\n    LIMIT {top_k};\n    \"\"\"\n    # Execute the query\n    cursor.execute(query_sql)\n    results = cursor.fetchall()\n\n    return results\n```\n\n\n```python\nquery_results = query_knn(\"modern art in Europe\", \"Articles\")\nfor i, result in enumerate(query_results):\n    print(f\"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})\")\n```\n\n    1. Museum of Modern Art (Score: 0.501)\n    2. Western Europe (Score: 0.485)\n    3. Renaissance art (Score: 0.479)\n    4. Pop art (Score: 0.472)\n    5. Northern Europe (Score: 0.461)\n    6. Hellenistic art (Score: 0.458)\n    7. Modernist literature (Score: 0.447)\n    8. Art film (Score: 0.44)\n    9. Central Europe (Score: 0.439)\n    10. Art (Score: 0.437)\n    11. European (Score: 0.437)\n    12. Byzantine art (Score: 0.436)\n    13. Postmodernism (Score: 0.435)\n    14. Eastern Europe (Score: 0.433)\n    15. Cubism (Score: 0.433)\n    16. Europe (Score: 0.432)\n    17. Impressionism (Score: 0.432)\n    18. Bauhaus (Score: 0.431)\n    19. Surrealism (Score: 0.429)\n    20. Expressionism (Score: 0.429)"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a21902d1-a625-4700-a2c4-4d512be6953e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a21902d1-a625-4700-a2c4-4d512be6953e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f5de0968-bddc-4432-9ff2-c75ee4d2c664\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1b740b85-289c-418f-8cdc-f5215bac8f5d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb\", \"title\": \"Using Hologres as a vector database for OpenAI embeddings\", \"tokens\": 4831, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3764d46578c48a12652a3f00210390d5f0082618a945b2c6a52da049ea388d96\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15528, \"end_char_idx\": 17388, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "ref_doc_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "chroma:document": "```python\nquery_results = query_knn(\"modern art in Europe\", \"Articles\")\nfor i, result in enumerate(query_results):\n    print(f\"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})\")\n```\n\n    1. Museum of Modern Art (Score: 0.501)\n    2. Western Europe (Score: 0.485)\n    3. Renaissance art (Score: 0.479)\n    4. Pop art (Score: 0.472)\n    5. Northern Europe (Score: 0.461)\n    6. Hellenistic art (Score: 0.458)\n    7. Modernist literature (Score: 0.447)\n    8. Art film (Score: 0.44)\n    9. Central Europe (Score: 0.439)\n    10. Art (Score: 0.437)\n    11. European (Score: 0.437)\n    12. Byzantine art (Score: 0.436)\n    13. Postmodernism (Score: 0.435)\n    14. Eastern Europe (Score: 0.433)\n    15. Cubism (Score: 0.433)\n    16. Europe (Score: 0.432)\n    17. Impressionism (Score: 0.432)\n    18. Bauhaus (Score: 0.431)\n    19. Surrealism (Score: 0.429)\n    20. Expressionism (Score: 0.429)\n\n\n\n```python\n# This time we'll query using content vector\nquery_results = query_knn(\"Famous battles in Scottish history\", \"Articles\", \"content_vector\")\nfor i, result in enumerate(query_results):\n    print(f\"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})\")\n```\n\n    1. Battle of Bannockburn (Score: 0.489)\n    2. Wars of Scottish Independence (Score: 0.474)\n    3. 1651 (Score: 0.457)\n    4. First War of Scottish Independence (Score: 0.452)\n    5. Robert I of Scotland (Score: 0.445)\n    6. 841 (Score: 0.441)\n    7. 1716 (Score: 0.441)\n    8. 1314 (Score: 0.429)\n    9. 1263 (Score: 0.428)\n    10. William Wallace (Score: 0.426)\n    11. Stirling (Score: 0.419)\n    12. 1306 (Score: 0.419)\n    13. 1746 (Score: 0.418)\n    14. 1040s (Score: 0.414)\n    15. 1106 (Score: 0.412)\n    16. 1304 (Score: 0.411)\n    17. David II of Scotland (Score: 0.408)\n    18. Braveheart (Score: 0.407)\n    19. 1124 (Score: 0.406)\n    20. July 27 (Score: 0.405)\n\n\n\n```python\n\n```"}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: b935e0cf-20c0-4a23-b100-21f016a57adb
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"b935e0cf-20c0-4a23-b100-21f016a57adb\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7c54ae26-1c8c-487f-bf17-dd7249ab7b6c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"056e2616-a5d4-4255-96a8-bba569b14500\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"da3a6cc971976443e5042af317c47553534dcf064581159176f061bf697d0812\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3150, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "ref_doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "chroma:document": "# Using Chroma for Embeddings Search\n\nThis notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.\n\n### What is a Vector Database\n\nA vector database is a database made to store, manage and search embedding vectors. The use of embeddings to encode unstructured data (text, audio, video and more) as vectors for consumption by machine-learning models has exploded in recent years, due to the increasing effectiveness of AI in solving use cases involving natural language, image recognition and other unstructured forms of data. Vector databases have emerged as an effective solution for enterprises to deliver and scale these use cases.\n\n### Why use a Vector Database\n\nVector databases enable enterprises to take many of the embeddings use cases we've shared in this repo (question and answering, chatbot and recommendation services, for example), and make use of them in a secure, scalable environment. Many of our customers make embeddings solve their problems at small scale but performance and security hold them back from going into production - we see vector databases as a key component in solving that, and in this guide we'll walk through the basics of embedding text data, storing it in a vector database and using it for semantic search.\n\n\n### Demo Flow\nThe demo flow is:\n- **Setup**: Import packages and set any required variables\n- **Load data**: Load a dataset and embed it using OpenAI embeddings\n- **Chroma**:\n    - *Setup*: Here we'll set up the Python client for Chroma. For more details go [here](https://docs.trychroma.com/usage-guide)\n    - *Index Data*: We'll create collections with vectors for __titles__ and __content__\n    - *Search Data*: We'll run a few searches to confirm it works\n\nOnce you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings.\n\n## Setup\n\nImport the required libraries and set the embedding model that we'd like to use.\n\n\n```python\n# Make sure the OpenAI library is installed\n%pip install openai\n\n# We'll need to install the Chroma client\n%pip install chromadb\n\n# Install wget to pull zip file\n%pip install wget\n\n# Install numpy for data manipulation\n%pip install numpy\n```\n\n    Collecting openai\n      Obtaining dependency information for openai from https://files.pythonhosted.org/packages/67/78/7588a047e458cb8075a4089d721d7af5e143ff85a2388d4a28c530be0494/openai-0.27.8-py3-none-any.whl.metadata\n      Downloading openai-0.27.8-py3-none-any.whl.metadata (13 kB)\n    Collecting requests>=2.20 (from openai)\n      Obtaining dependency information for requests>=2.20 from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata\n      Using cached requests-2.31.0-py3-none-any.whl.metadata (4."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 056e2616-a5d4-4255-96a8-bba569b14500
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"056e2616-a5d4-4255-96a8-bba569b14500\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7c54ae26-1c8c-487f-bf17-dd7249ab7b6c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b935e0cf-20c0-4a23-b100-21f016a57adb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"712d163f1c2c1cb788a63e3fcd3e1b2fedba0d41cdc9668c9503892a6a6ec121\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"66597835-989b-42d1-907a-969fec381a64\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"494e907ac46e0d980cfdd6b423c46a7b1e30acae9fd28fc6e9a1338e69d45b32\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2287, \"end_char_idx\": 4380, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "ref_doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "chroma:document": "```python\n# Make sure the OpenAI library is installed\n%pip install openai\n\n# We'll need to install the Chroma client\n%pip install chromadb\n\n# Install wget to pull zip file\n%pip install wget\n\n# Install numpy for data manipulation\n%pip install numpy\n```\n\n    Collecting openai\n      Obtaining dependency information for openai from https://files.pythonhosted.org/packages/67/78/7588a047e458cb8075a4089d721d7af5e143ff85a2388d4a28c530be0494/openai-0.27.8-py3-none-any.whl.metadata\n      Downloading openai-0.27.8-py3-none-any.whl.metadata (13 kB)\n    Collecting requests>=2.20 (from openai)\n      Obtaining dependency information for requests>=2.20 from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata\n      Using cached requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)\n    Collecting tqdm (from openai)\n      Using cached tqdm-4.65.0-py3-none-any.whl (77 kB)\n    Collecting aiohttp (from openai)\n      Obtaining dependency information for aiohttp from https://files.pythonhosted.org/packages/fa/9e/49002fde2a97d7df0e162e919c31cf13aa9f184537739743d1239edd0e67/aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata\n      Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata (7.7 kB)\n    Collecting charset-normalizer<4,>=2 (from requests>=2.20->openai)\n      Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/ec/a7/96835706283d63fefbbbb4f119d52f195af00fc747e67cc54397c56312c8/charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata\n      Using cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (31 kB)\n    Collecting idna<4,>=2.5 (from requests>=2.20->openai)\n      Using cached idna-3.4-py3-none-any.whl (61 kB)\n    Collecting urllib3<3,>=1.21.1 (from requests>=2.20->openai)\n      Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/9b/81/62fd61001fa4b9d0df6e31d47ff49cfa9de4af03adecf339c7bc30656b37/urllib3-2.0.4-py3-none-any.whl."}


created_at: 2024-08-26 20:31:21
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 66597835-989b-42d1-907a-969fec381a64
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"66597835-989b-42d1-907a-969fec381a64\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7c54ae26-1c8c-487f-bf17-dd7249ab7b6c\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"056e2616-a5d4-4255-96a8-bba569b14500\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb\", \"title\": \"Using Chroma for Embeddings Search\", \"tokens\": 10702, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"fe463b61c44a8eb3af0bd04df171fcbf862e4032eaaf99ad7737b516c255ce93\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8eba3e05-0831-4436-a2c1-3a7a6b057c6a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3a85adff5038cd5128ba3a32660df4e898822687a4e911a8aa7b363160bb8c35\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3456, \"end_char_idx\": 5404, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "ref_doc_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "chroma:document": "8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata\n      Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata (7.7 kB)\n    Collecting charset-normalizer<4,>=2 (from requests>=2.20->openai)\n      Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/ec/a7/96835706283d63fefbbbb4f119d52f195af00fc747e67cc54397c56312c8/charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata\n      Using cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (31 kB)\n    Collecting idna<4,>=2.5 (from requests>=2.20->openai)\n      Using cached idna-3.4-py3-none-any.whl (61 kB)\n    Collecting urllib3<3,>=1.21.1 (from requests>=2.20->openai)\n      Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/9b/81/62fd61001fa4b9d0df6e31d47ff49cfa9de4af03adecf339c7bc30656b37/urllib3-2.0.4-py3-none-any.whl.metadata\n      Downloading urllib3-2.0.4-py3-none-any.whl.metadata (6.6 kB)\n    Collecting certifi>=2017.4.17 (from requests>=2.20->openai)\n      Using cached certifi-2023.5.7-py3-none-any.whl (156 kB)\n    Collecting attrs>=17.3.0 (from aiohttp->openai)\n      Using cached attrs-23.1.0-py3-none-any.whl (61 kB)\n    Collecting multidict<7.0,>=4.5 (from aiohttp->openai)\n      Using cached multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl (29 kB)\n    Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->openai)\n      Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n    Collecting yarl<2.0,>=1.0 (from aiohttp->openai)\n      Using cached yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl (62 kB)\n    Collecting frozenlist>=1.1.1 (from aiohttp->openai)\n      Obtaining dependency information for frozenlist>=1.1.1 from https://files.pythonhosted.org/packages/67/6a/55a49da0fa373ac9aa49ccd5b6393ecc183e2a0904d9449ea3ee1163e0b1/frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata\n      Downloading frozenlist-1.4."}


