# embedding_metadata

key: _node_content
string_value:
{"id_": "6f024d8c-b62d-495c-9132-f2332822c0ba", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3f1816d9-abcc-4257-94e4-c37ba5b2d8bd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c63fc06b32b2f4bbd69cc17ae81658ce47834e5c36e8abaa034a9d1e5a839279", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "49797d37-c323-421e-aa78-4d7a0048b357", "node_type": "1", "metadata": {}, "hash": "a15ed8af85624a429687178daef5c2c1a08c97797cee34d9556a4c8c6941ee1d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 25168, "end_char_idx": 27705, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.50</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">26</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">02</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">2024</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">, </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">17:58</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Models - OpenAI API</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">MODEL</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">DE S CRIPTION</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">tts-</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">1</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">New  Text-to-speech </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">1</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The latest tex...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


key: _node_content
string_value:
{"id_": "49797d37-c323-421e-aa78-4d7a0048b357", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6f024d8c-b62d-495c-9132-f2332822c0ba", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "18211ea07077b7649e7ae01f99e83a6f7a18d1a399a1df2c8e3977dfff9fd7ea", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "98dc1dc7-8039-4e82-aa31-84eafbe0d027", "node_type": "1", "metadata": {}, "hash": "19fd98d37c86cb6136cacb44fe66023c9cf3b2a3ea0b9aea05098cae62526de6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 27708, "end_char_idx": 29595, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.44</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">26</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">02</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">2024</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">, </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">17:58</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Models - OpenAI API</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">ENDP OINT</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">DATA USED</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">FOR TRAINING</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">DEFAULT</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">RETENTION</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">ELIGIBLE FO...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "98dc1dc7-8039-4e82-aa31-84eafbe0d027", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "49797d37-c323-421e-aa78-4d7a0048b357", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "45c89448631927c5d470fbda5547ca5fae1d350f6b98b4ba9b164402f379ac1f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a26cc548-831f-43b2-9404-057551cedbaf", "node_type": "1", "metadata": {}, "hash": "d86cdf3a11a6bb310918e5aab51b22ba7ca96924281cb1c3f4cdf92b859f1cc7", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 29589, "end_char_idx": 32043, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">Yes, the Whisper model is a general-purpose speech recognition model mentioned in the content, capable of </span>
<span style="color: #00875f; text-decoration-color: #00875f">multilingual speech recognition, speech translation, and language identification. The v2-large model, referred to </span>
<span style="color: #00875f; text-decoration-color: #00875f">as </span><span style="color: #00875f; text-decoration-color: #00875f">"whisper-1"</span><span style="color: #00875f; text-decoration-color: #00875f">, is available through an API and is optimized for faster performance.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Which embedding model should I use for non-English use cases?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.57</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "a26cc548-831f-43b2-9404-057551cedbaf", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "98dc1dc7-8039-4e82-aa31-84eafbe0d027", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "20b27efa6aaf05b8795e44cc3d7da00051eaf1b721743e0e34e6e74bebbf3330", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ad50e971-357d-4a46-a987-0258023319bb", "node_type": "1", "metadata": {}, "hash": "b5aff794b9108b65dde78bd3ca80ceda270860ff317ccbcde9269b5054f788bc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 30865, "end_char_idx": 32400, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.57</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.46</span>
</pre>


key: _node_content
string_value:
{"id_": "ad50e971-357d-4a46-a987-0258023319bb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a26cc548-831f-43b2-9404-057551cedbaf", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "70ba63a72185e223dc0eb2152be19ac2ababb7a0f0e26af228cf9f4bbcbe5cdc", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "96999881-08fe-47f3-a35b-31da212bea38", "node_type": "1", "metadata": {}, "hash": "cad3e2b0b03ca2adbb162077da1a846065b401d6069eaae2694bc1c9b666ef12", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 31484, "end_char_idx": 34021, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.46</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">26</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">02</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">2024</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">, </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">17:58</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Models - OpenAI API</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">MODEL</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">DE S CRIPTION</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">tts-</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">1</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">New  Text-to-speech </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">1</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">The latest tex...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


key: _node_content
string_value:
{"id_": "96999881-08fe-47f3-a35b-31da212bea38", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ad50e971-357d-4a46-a987-0258023319bb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "50d061b7279b236abdd807eeac47b57ae84d17b467134a0d33e884a2ca57228f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c35ab626-caf3-432e-96cd-8be88564a858", "node_type": "1", "metadata": {}, "hash": "0aac3eae59addae4e703accba947b48b023bc827ac42c7e85ffe75a09d41d92c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 34024, "end_char_idx": 35794, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.40</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">26</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">02</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">2024</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">, </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">17:58</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Models - OpenAI API</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Multilingual capabilities</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">GPT-</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">4</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f"> outperforms both previous larg...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "c35ab626-caf3-432e-96cd-8be88564a858", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "96999881-08fe-47f3-a35b-31da212bea38", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fe8c40853018bed9ee06c2f87cae881850071e51212794eb5235b79932491997", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e92a1866-2d77-44a6-a285-a1def69afe9d", "node_type": "1", "metadata": {}, "hash": "aa9077f13241d95863161de656c028f4d1a2e0aebe3671457fac577b59bde19e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 35788, "end_char_idx": 37795, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">For non-English use cases, you should use the </span><span style="color: #00875f; text-decoration-color: #00875f">"V3 large"</span><span style="color: #00875f; text-decoration-color: #00875f"> embedding model, as it is described as the most capable </span>
<span style="color: #00875f; text-decoration-color: #00875f">for both English and non-English tasks, with an output dimension of </span><span style="color: #00875f; text-decoration-color: #00875f; font-weight: bold">3</span><span style="color: #00875f; text-decoration-color: #00875f">,</span><span style="color: #00875f; text-decoration-color: #00875f; font-weight: bold">072</span><span style="color: #00875f; text-decoration-color: #00875f">.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Can I introduce new knowledge in my LLM app using RAG?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.50</span>
</pre>


key: _node_content
string_value:
{"id_": "e92a1866-2d77-44a6-a285-a1def69afe9d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c35ab626-caf3-432e-96cd-8be88564a858", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "1936b94236d2a213c52cf9817b2014b78dacfb6c99a36498f525eb90edd0cdce", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f1ee7469-1fd3-4348-8abc-d5374be937fb", "node_type": "1", "metadata": {}, "hash": "9b2f55dd8c2c238cf83b7a7520048ec39b972a8e3633846abd80328cfa5196e9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 36817, "end_char_idx": 38852, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Can I introduce new knowledge in my LLM app using RAG?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.50</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What is RAG</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Retrieve information to Augment the model’s knowledge and Generate the output</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">“What is y...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.49</span>
</pre>


key: _node_content
string_value:
{"id_": "f1ee7469-1fd3-4348-8abc-d5374be937fb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e92a1866-2d77-44a6-a285-a1def69afe9d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "d6fedc813325885a04d26334a49f16fb4f390520a39014efa4593f9b91cedf5d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dd4ccde5-c84c-4786-865b-f5c56b70a565", "node_type": "1", "metadata": {}, "hash": "f9eb8763ba491b4ae4c80a7b6363ff234a6dffa7a5f2de3d1857b5f5814db18a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 37800, "end_char_idx": 39892, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What is RAG</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Retrieve information to Augment the model’s knowledge and Generate the output</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">“What is y...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.49</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">When to use RAG</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Good for  ✅</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Not good for  ❌</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Introducing new information to the model</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Teaching ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "dd4ccde5-c84c-4786-865b-f5c56b70a565", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f1ee7469-1fd3-4348-8abc-d5374be937fb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2b1bb21b3de80c25dfdb56468d9bfde6c5aedc8c4f25ff93d1334ad36657b303", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b037dd33-1d48-430b-aaf7-6333a9b49df1", "node_type": "1", "metadata": {}, "hash": "b79afdf59a8853970e405a3d57258838df60ebabbe0c5c0d2f763efa99370f8e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 38857, "end_char_idx": 41017, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">When to use RAG</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Good for  ✅</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Not good for  ❌</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Introducing new information to the model</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Teaching ...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.43</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Data preparation: augmenting content</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What does “Augmentingcontent” mean?</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Augmenti...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "b037dd33-1d48-430b-aaf7-6333a9b49df1", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dd4ccde5-c84c-4786-865b-f5c56b70a565", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2aacc2d97f7103a2bc2fd4fa4b978ab223afe8966f134fe952ef5b0f71a60591", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca", "node_type": "1", "metadata": {}, "hash": "a37011a59e753a8bd76b2629edb02a625e2ecb35b443836a67aeeb82905146af", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 39886, "end_char_idx": 42238, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.43</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Data preparation: augmenting content</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What does “Augmentingcontent” mean?</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Augmenti...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>
<span style="color: #00875f; text-decoration-color: #00875f">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> How many examples do I need to fine-tune a model?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>


key: _node_content
string_value:
{"id_": "eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b037dd33-1d48-430b-aaf7-6333a9b49df1", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "177ee2b3443b01c80137f449fcf9c122e6c96be44980e040707be3923bca0c0e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5c0debe7-9932-4a1c-9a28-29c7737f1a95", "node_type": "1", "metadata": {}, "hash": "914aac184da76168151c5f196379d55ca66060490f4b231b01ea31cb7426e2f4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 41011, "end_char_idx": 43499, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>
<span style="color: #00875f; text-decoration-color: #00875f">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> How many examples do I need to fine-tune a model?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.68</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What is Fine-tuning</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Public Model</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Training data</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Training</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tunedmodel</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning a model consists...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "5c0debe7-9932-4a1c-9a28-29c7737f1a95", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "eea1fa40-2dbd-4fe9-9d1e-d2d219cb8cca", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3fe3391864b70a2b8ca86c88f09bda1b292636cc18b731bd2963d2bbc1807da5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d690c483-ad6e-4d3f-897e-21f333467e1a", "node_type": "1", "metadata": {}, "hash": "5f6fe07088436f8c23e3b357fde7ec514c72b794553450952763bc7d44ed6c4e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 42600, "end_char_idx": 44624, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What is Fine-tuning</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Public Model</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Training data</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Training</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tunedmodel</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning a model consists...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.62</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">When to fine-tune</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning is good for:</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">- Following a given format or tone for the output</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">- Proce...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "d690c483-ad6e-4d3f-897e-21f333467e1a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5c0debe7-9932-4a1c-9a28-29c7737f1a95", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3385df3b82998fbd5d23793b036014ca7488decf815c8392a6b3efae47e0817b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "52d73bdb-e880-43f6-b4de-74ac764a1020", "node_type": "1", "metadata": {}, "hash": "813a2099892ca6bd73f0843192ae159e6c4a25eab17eb08d9e60716b336f19f2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 43493, "end_char_idx": 45613, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.62</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">When to fine-tune</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning is good for:</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">- Following a given format or tone for the output</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">- Proce...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.57</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Overview</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning involves adjusting theparameters of pre-trained models on aspeciﬁc dataset or t...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "52d73bdb-e880-43f6-b4de-74ac764a1020", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d690c483-ad6e-4d3f-897e-21f333467e1a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9c382b2042127d1d17fa1f641e1422444299a2e7ffb5f956b5b6da25ad770267", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f6aeb219-c8c9-4a52-9303-af0d2e37ed6d", "node_type": "1", "metadata": {}, "hash": "8bc9fa8ce34c4b65204984a98a574fe7924cdc583107ab9f09cf33e9683bf5f5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 44618, "end_char_idx": 47112, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.57</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Overview</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Fine-tuning involves adjusting theparameters of pre-trained models on aspeciﬁc dataset or t...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">We recommend using </span><span style="color: #00875f; text-decoration-color: #00875f; font-weight: bold">50</span><span style="color: #00875f; text-decoration-color: #00875f">-</span><span style="color: #00875f; text-decoration-color: #00875f; font-weight: bold">100</span><span style="color: #00875f; text-decoration-color: #00875f"> examples for fine-tuning a model, even though the minimum is </span><span style="color: #00875f; text-decoration-color: #00875f; font-weight: bold">10</span><span style="color: #00875f; text-decoration-color: #00875f">.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Which metric can I use to evaluate a summarization task?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>


key: _node_content
string_value:
{"id_": "f6aeb219-c8c9-4a52-9303-af0d2e37ed6d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "52d73bdb-e880-43f6-b4de-74ac764a1020", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "eadba6ef4e18c5cd2ac19b92c5c7788295e338bd5f27fe251568001664294b23", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e4c79905-7592-4334-bb0d-74f6018ecf46", "node_type": "1", "metadata": {}, "hash": "23b24f5646d126a18d73f7e318fad42e04cd46b401b6a23d47e75eb3067331cb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 46489, "end_char_idx": 48526, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Which metric can I use to evaluate a summarization task?</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.53</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">ROUGE is a common metric for evaluating machine summariz...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.49</span>
</pre>


key: _node_content
string_value:
{"id_": "e4c79905-7592-4334-bb0d-74f6018ecf46", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f6aeb219-c8c9-4a52-9303-af0d2e37ed6d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "66552fdbd9486b66a2d806e1c1342b95ec73899fc55cc14af65e0f53ff5c55a2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f0bda61b-73e3-4f9b-8dd2-290130eac375", "node_type": "1", "metadata": {}, "hash": "37be058a33be894c3ed59e93a7b5323b801b43e93f6f91c8b87d23bcfda110df", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 47474, "end_char_idx": 49498, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">ROUGE is a common metric for evaluating machine summariz...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.49</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Component evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Subjective evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Compari...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "f0bda61b-73e3-4f9b-8dd2-290130eac375", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e4c79905-7592-4334-bb0d-74f6018ecf46", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "cc8959cc8ee11e4957b8608eca1d57eabcb3fb45f2082b4719306c32f22cfa28", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "37adb2cd-8729-40ed-955f-121e02f22a37", "node_type": "1", "metadata": {}, "hash": "b7b0386d7119179de071a83afe5d53fc654116e61ea983d1f66847886f79e47d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 48531, "end_char_idx": 50555, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Component evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Subjective evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Compari...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.48</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">BLEU score is another standard metric, this time focusin...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "37adb2cd-8729-40ed-955f-121e02f22a37", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f0bda61b-73e3-4f9b-8dd2-290130eac375", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "046191cdd9e518f6a0d0edb7d41b3053a83262d524008a1e8bc97ea56d31ec59", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d30dbb8a-6424-4f11-8e11-93ed84be995c", "node_type": "1", "metadata": {}, "hash": "d64ff46d20f88f1726d61f508953d8c11f8b54905fa9d997c4ea17018821e4c2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 49492, "end_char_idx": 51703, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.48</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">BLEU score is another standard metric, this time focusin...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">ROUGE is a common metric you can use to evaluate a summarization task.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>
<span style="color: #af005f; text-decoration-color: #af005f">ground truth.</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>


key: _node_content
string_value:
{"id_": "d30dbb8a-6424-4f11-8e11-93ed84be995c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "37adb2cd-8729-40ed-955f-121e02f22a37", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ac20df63878a4856a3359deb24ab3706381b0a4a86702a9f541b8458ac6ca213", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4", "node_type": "1", "metadata": {}, "hash": "21ae55db8ab4fe75d8d4c228c3166894aceaca747fd5523c99f1f54151f08420", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 50549, "end_char_idx": 52828, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">ROUGE is a common metric you can use to evaluate a summarization task.</span>

--------------


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #af005f; text-decoration-color: #af005f; font-weight: bold">QUERY:</span><span style="color: #af005f; text-decoration-color: #af005f"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>
<span style="color: #af005f; text-decoration-color: #af005f">ground truth.</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">Matching content:</span>

</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.60</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What are evals</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Example</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Our ground truth matches the predicted answer, so the evaluation passes!</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Eval...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d30dbb8a-6424-4f11-8e11-93ed84be995c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b5b37d97d2654c190633b769f6b25abc4296f35dfa3571c0b2237bdf36f01108", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "97211244-1d04-4b01-bf52-36853a83348f", "node_type": "1", "metadata": {}, "hash": "390f2232ed3c77cd6474f0ef027e27972a09866bb451bc2ceade6168bd1fd7fa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 51708, "end_char_idx": 53885, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.60</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What are evals</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Example</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Our ground truth matches the predicted answer, so the evaluation passes!</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Eval...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.59</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What are evals</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Example</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">An evaluation contains a question and a correct answer. We call this the grou...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "97211244-1d04-4b01-bf52-36853a83348f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fd5eaad1-3b31-43ed-aee7-fb53b2a95cf4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "e3b2eeb2334c2a274b200dc6d88b14d02268b8f22c8387cbd19535ceb5cc54da", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "06d75069-f6eb-4f39-b0de-53e40523a777", "node_type": "1", "metadata": {}, "hash": "ee47e9d7e2d52f10d301ada39b25dee618d51c492b9037c5df1b728a76863c70", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 52822, "end_char_idx": 54242, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.59</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What are evals</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Example</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">An evaluation contains a question and a correct answer. We call this the grou...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.50</span>
</pre>


key: _node_content
string_value:
{"id_": "06d75069-f6eb-4f39-b0de-53e40523a777", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "97211244-1d04-4b01-bf52-36853a83348f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "dc0179d3445632ab59ccee7fb7390e27bfac8a72ee4c88ce672e2658d11ed2aa", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c314e1ec-5c98-4532-8a93-4ab5acc8ac1b", "node_type": "1", "metadata": {}, "hash": "5e2796b03cc5e9a22146d789ca69b93e8a03a016d867cad6ab570ddb10c24641", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 53190, "end_char_idx": 55214, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What are evals</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Example</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">An evaluation contains a question and a correct answer. We call this the grou...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic">Similarity: </span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic">0.50</span>
</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What they’re good for</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What to be aware of</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">A good sta...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>


key: _node_content
string_value:
{"id_": "c314e1ec-5c98-4532-8a93-4ab5acc8ac1b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "06d75069-f6eb-4f39-b0de-53e40523a777", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4e28c927ad2fc669e97033fd3e6d089249065c890ed09daf48eb0e8c24c5dbb5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d1c961c2-db90-417a-bfca-d9b43895fcea", "node_type": "1", "metadata": {}, "hash": "2d613399576b84432d72fe3e3edb284de0bc4ac7d4badf03828a130325b0468c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 54247, "end_char_idx": 57652, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Technical patterns</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">Metric-based evaluations</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What they’re good for</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">What to be aware of</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">●</span>
<span style="color: #5f5f5f; text-decoration-color: #5f5f5f">A good sta...</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">[</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f">/</span><span style="color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold">]</span>


</pre>




<pre style="white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace"><span style="color: #008787; text-decoration-color: #008787; font-weight: bold">REPLY:</span>

<span style="color: #00875f; text-decoration-color: #00875f">The content provided is relevant and offers a detailed example for an evaluation process comparing to a ground </span>
<span style="color: #00875f; text-decoration-color: #00875f">truth. Here's a concise explanation based on the content:</span>

<span style="color: #00875f; text-decoration-color: #00875f">In the given example, the evaluation process involves a question-and-answer scenario to verify the accuracy of </span>
<span style="color: #00875f; text-decoration-color: #00875f">information retrieved by a tool or system in response to a query. The question posed is, </span><span style="color: #00875f; text-decoration-color: #00875f">"What is the population of</span>
<span style="color: #00875f; text-decoration-color: #00875f">Canada?"</span><span style="color: #00875f; text-decoration-color: #00875f"> The ground truth, or the correct answer, is established as </span><span style="color: #00875f; text-decoration-color: #00875f">"The population of Canada in 2023 is 39,566,248</span>
<span style="color: #00875f; text-decoration-color: #00875f">people."</span><span style="color: #00875f; text-decoration-color: #00875f"> A tool labeled </span><span style="color: #00875f; text-decoration-color: #00875f">"LLM"</span><span style="color: #00875f; text-decoration-color: #00875f"> is then used to search for the answer, which predicts </span><span style="color: #00875f; text-decoration-color: #00875f">"The current population of </span>
<span style="color: #00875f; text-decoration-color: #00875f">Canada is 39,566,248 as of Tuesday, May 23, 2023."</span><span style="color: #00875f; text-decoration-color: #00875f"> This predicted answer matches the ground truth exactly, </span>
<span style="color: #00875f; text-decoration-color: #00875f">indicating that the evaluation passes. This process demonstrates how an evaluation can be used to verify the </span>
<span style="color: #00875f; text-decoration-color: #00875f">accuracy of information retrieved by a tool, comparing the predicted answer to the ground truth to ensure </span>
<span style="color: #00875f; text-decoration-color: #00875f">correctness.</span>

--------------


key: _node_content
string_value:
{"id_": "d1c961c2-db90-417a-bfca-d9b43895fcea", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "db6ae619-b33c-4db0-bb24-9433261dcfbc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "230396899f7488cfa57ab447ee2a55fb5918d89b159c2f37bfc35c893a6139c4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c314e1ec-5c98-4532-8a93-4ab5acc8ac1b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "72e35f0560759578ef7d94b7dcbe83d6cce08b02cf06768c1ecb5a4c4ce43e35", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 57655, "end_char_idx": 58851, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</pre>



## Wrapping up

In this notebook, we have learned how to develop a basic RAG pipeline based on PDF documents. This includes:

- How to parse pdf documents, taking slide decks and an export from an HTML page as examples, using a python library as well as GPT-4V to interpret the visuals
- How to process the extracted content, clean it and chunk it into several pieces
- How to embed the processed content using OpenAI embeddings
- How to retrieve content that is relevant to an input query
- How to use GPT-4-turbo to generate an answer using the retrieved content as context

If you want to explore further, consider these optimisations:

- Playing around with the prompts provided as examples
- Chunking the content further and adding metadata as context to each chunk
- Adding rule-based filtering on the retrieval results or re-ranking results to surface to most relevant content

You can apply the techniques covered in this notebook to multiple use cases, such as assistants that can access your proprietary data, customer service or FAQ bots that can read from your internal policies, or anything that requires leveraging rich documents that would be better understood as images.


key: _node_content
string_value:
{"id_": "787eb8d2-57ea-4c3e-9651-752be093e7de", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e39f8626-06f4-43b5-b812-987468392af2", "node_type": "1", "metadata": {}, "hash": "ceff499f9b12eb7e39fc76d174c27b66b86100964ea46e7dd7a7b3bcec56cba4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3493, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Code search using embeddings

This notebook shows how Ada embeddings can be used to implement semantic code search. For this demonstration, we use our own [openai-python code repository](https://github.com/openai/openai-python). We implement a simple version of file parsing and extracting of functions from python files, which can be embedded, indexed, and queried.

### Helper Functions

We first setup some simple parsing functions that allow us to extract important information from our codebase.


```python
import pandas as pd
from pathlib import Path

DEF_PREFIXES = ['def ', 'async def ']
NEWLINE = '\n'

def get_function_name(code):
    """
    Extract function name from a line beginning with 'def' or 'async def'.
    """
    for prefix in DEF_PREFIXES:
        if code.startswith(prefix):
            return code[len(prefix): code.index('(')]


def get_until_no_space(all_lines, i):
    """
    Get all lines until a line outside the function definition is found.
    """
    ret = [all_lines[i]]
    for j in range(i + 1, len(all_lines)):
        if len(all_lines[j]) == 0 or all_lines[j][0] in [' ', '\t', ')']:
            ret.append(all_lines[j])
        else:
            break
    return NEWLINE.join(ret)


def get_functions(filepath):
    """
    Get all functions in a Python file.
    """
    with open(filepath, 'r') as file:
        all_lines = file.read().replace('\r', NEWLINE).split(NEWLINE)
        for i, l in enumerate(all_lines):
            for prefix in DEF_PREFIXES:
                if l.startswith(prefix):
                    code = get_until_no_space(all_lines, i)
                    function_name = get_function_name(code)
                    yield {
                        'code': code,
                        'function_name': function_name,
                        'filepath': filepath,
                    }
                    break


def extract_functions_from_repo(code_root):
    """
    Extract all .py functions from the repository.
    """
    code_files = list(code_root.glob('**/*.py'))

    num_files = len(code_files)
    print(f'Total number of .py files: {num_files}')

    if num_files == 0:
        print('Verify openai-python repo exists and code_root is set correctly.')
        return None

    all_funcs = [
        func
        for code_file in code_files
        for func in get_functions(str(code_file))
    ]

    num_funcs = len(all_funcs)
    print(f'Total number of functions extracted: {num_funcs}')

    return all_funcs
```

# Data Loading

We'll first load the openai-python folder and extract the needed information using the functions we defined above.


```python
# Set user root directory to the 'openai-python' repository
root_dir = Path.home()

# Assumes the 'openai-python' repository exists in the user's root directory
code_root = root_dir / 'openai-python'

# Extract all functions from the repository
all_funcs = extract_functions_from_repo(code_root)
```

    Total number of .py files: 51
    Total number of functions extracted: 97


Now that we have our content, we can pass the data to the `text-embedding-3-small` model and get back our vector embeddings.


```python
from utils.embeddings_utils import get_embedding

df = pd.DataFrame(all_funcs)
df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, model='text-embedding-3-small'))
df['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))
df.to_csv("data/code_search_openai-python.csv", index=False)
df.head()
```


key: _node_content
string_value:
{"id_": "e39f8626-06f4-43b5-b812-987468392af2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "787eb8d2-57ea-4c3e-9651-752be093e7de", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3eb4c330b3a4446bbca3a1faa691fed60a5122f219013cc76ce1ae3818c72983", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "48706980-2642-4874-91fa-3822f254012d", "node_type": "1", "metadata": {}, "hash": "772462d4fab00bfd7cb7d9837b5a09685351944f0bc8a3dfbfb84c11d67b1283", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1883, "end_char_idx": 5101, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
def extract_functions_from_repo(code_root):
    """
    Extract all .py functions from the repository.
    """
    code_files = list(code_root.glob('**/*.py'))

    num_files = len(code_files)
    print(f'Total number of .py files: {num_files}')

    if num_files == 0:
        print('Verify openai-python repo exists and code_root is set correctly.')
        return None

    all_funcs = [
        func
        for code_file in code_files
        for func in get_functions(str(code_file))
    ]

    num_funcs = len(all_funcs)
    print(f'Total number of functions extracted: {num_funcs}')

    return all_funcs
```

# Data Loading

We'll first load the openai-python folder and extract the needed information using the functions we defined above.


```python
# Set user root directory to the 'openai-python' repository
root_dir = Path.home()

# Assumes the 'openai-python' repository exists in the user's root directory
code_root = root_dir / 'openai-python'

# Extract all functions from the repository
all_funcs = extract_functions_from_repo(code_root)
```

    Total number of .py files: 51
    Total number of functions extracted: 97


Now that we have our content, we can pass the data to the `text-embedding-3-small` model and get back our vector embeddings.


```python
from utils.embeddings_utils import get_embedding

df = pd.DataFrame(all_funcs)
df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, model='text-embedding-3-small'))
df['filepath'] = df['filepath'].map(lambda x: Path(x).relative_to(code_root))
df.to_csv("data/code_search_openai-python.csv", index=False)
df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>code</th>
      <th>function_name</th>
      <th>filepath</th>
      <th>code_embedding</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>def _console_log_level():\n    if openai.log i...</td>
      <td>_console_log_level</td>
      <td>openai/util.py</td>
      <td>[0.005937571171671152, 0.05450401455163956, 0....</td>
    </tr>
    <tr>
      <th>1</th>
      <td>def log_debug(message, **params):\n    msg = l...</td>
      <td>log_debug</td>
      <td>openai/util.py</td>
      <td>[0.017557814717292786, 0.05647840350866318, -0...</td>
    </tr>
    <tr>
      <th>2</th>
      <td>def log_info(message, **params):\n    msg = lo...</td>
      <td>log_info</td>
      <td>openai/util.py</td>
      <td>[0.022524144500494003, 0.06219055876135826, -0...</td>
    </tr>
    <tr>
      <th>3</th>
      <td>def log_warn(message, **params):\n    msg = lo...</td>
      <td>log_warn</td>
      <td>openai/util.py</td>
      <td>[0.030524108558893204, 0.0667714849114418, -0....</td>
    </tr>
    <tr>
      <th>4</th>
      <td>def logfmt(props):\n    def fmt(key, val):\n  ...</td>
      <td>logfmt</td>
      <td>openai/util.py</td>
      <td>[0.05337328091263771, 0.03697286546230316, -0....</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "48706980-2642-4874-91fa-3822f254012d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e39f8626-06f4-43b5-b812-987468392af2", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "044868805aa3c8153df80f541e7c9f691519660c7750903c2f07080b46571853", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fa51e041-9319-4639-93da-7945b3abe3d8", "node_type": "1", "metadata": {}, "hash": "6b4275a7284796c3380c98c21c2edf7d70f67550d62a280412fa549c0657d8c4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5105, "end_char_idx": 7982, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Testing

Let's test our endpoint with some simple queries. If you're familiar with the `openai-python` repository, you'll see that we're able to easily find functions we're looking for only a simple English description.

We define a search_functions method that takes our data that contains our embeddings, a query string, and some other configuration options. The process of searching our database works like such:

1. We first embed our query string (code_query) with `text-embedding-3-small`. The reasoning here is that a query string like 'a function that reverses a string' and a function like 'def reverse(string): return string[::-1]' will be very similar when embedded.
2. We then calculate the cosine similarity between our query string embedding and all data points in our database. This gives a distance between each point and our query.
3. We finally sort all of our data points by their distance to our query string and return the number of results requested in the function parameters. 


```python
from utils.embeddings_utils import cosine_similarity

def search_functions(df, code_query, n=3, pprint=True, n_lines=7):
    embedding = get_embedding(code_query, model='text-embedding-3-small')
    df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))

    res = df.sort_values('similarities', ascending=False).head(n)

    if pprint:
        for r in res.iterrows():
            print(f"{r[1].filepath}:{r[1].function_name}  score={round(r[1].similarities, 3)}")
            print("\n".join(r[1].code.split("\n")[:n_lines]))
            print('-' * 70)

    return res
```


```python
res = search_functions(df, 'fine-tuning input data validation logic', n=3)
```

    openai/validators.py:format_inferrer_validator  score=0.453
    def format_inferrer_validator(df):
        """
        This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.
        It will also suggest to use ada and explain train/validation split benefits.
        """
        ft_type = infer_task_type(df)
        immediate_msg = None
    ----------------------------------------------------------------------
    openai/validators.py:infer_task_type  score=0.37
    def infer_task_type(df):
        """
        Infer the likely fine-tuning task type from the data
        """
        CLASSIFICATION_THRESHOLD = 3  # min_average instances of each class
        if sum(df.prompt.str.len()) == 0:
            return "open-ended generation"
    ----------------------------------------------------------------------
    openai/validators.py:apply_validators  score=0.369
    def apply_validators(
        df,
        fname,
        remediation,
        validators,
        auto_accept,
        write_out_file_func,
    ----------------------------------------------------------------------


key: _node_content
string_value:
{"id_": "fa51e041-9319-4639-93da-7945b3abe3d8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1e01053d-9de7-4cea-b339-6b0f73e9881f", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "37f21d4656babb3c9119cdfd8c5c477919baeccd41ebd4b12624865903007c19", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "48706980-2642-4874-91fa-3822f254012d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Code_search_using_embeddings.ipynb", "title": "Data Loading", "tokens": 2354, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "b1f7f9518da7102af02084d5740fd90ca8141aefd49d17a5fe5cb10ff12c6cce", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6733, "end_char_idx": 9958, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
res = search_functions(df, 'fine-tuning input data validation logic', n=3)
```

    openai/validators.py:format_inferrer_validator  score=0.453
    def format_inferrer_validator(df):
        """
        This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.
        It will also suggest to use ada and explain train/validation split benefits.
        """
        ft_type = infer_task_type(df)
        immediate_msg = None
    ----------------------------------------------------------------------
    openai/validators.py:infer_task_type  score=0.37
    def infer_task_type(df):
        """
        Infer the likely fine-tuning task type from the data
        """
        CLASSIFICATION_THRESHOLD = 3  # min_average instances of each class
        if sum(df.prompt.str.len()) == 0:
            return "open-ended generation"
    ----------------------------------------------------------------------
    openai/validators.py:apply_validators  score=0.369
    def apply_validators(
        df,
        fname,
        remediation,
        validators,
        auto_accept,
        write_out_file_func,
    ----------------------------------------------------------------------



```python
res = search_functions(df, 'find common suffix', n=2, n_lines=10)
```

    openai/validators.py:get_common_xfix  score=0.487
    def get_common_xfix(series, xfix="suffix"):
        """
        Finds the longest common suffix or prefix of all the values in a series
        """
        common_xfix = ""
        while True:
            common_xfixes = (
                series.str[-(len(common_xfix) + 1) :]
                if xfix == "suffix"
                else series.str[: len(common_xfix) + 1]
    ----------------------------------------------------------------------
    openai/validators.py:common_completion_suffix_validator  score=0.449
    def common_completion_suffix_validator(df):
        """
        This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.
        """
        error_msg = None
        immediate_msg = None
        optional_msg = None
        optional_fn = None
    
        ft_type = infer_task_type(df)
    ----------------------------------------------------------------------



```python
res = search_functions(df, 'Command line interface for fine-tuning', n=1, n_lines=20)
```

    openai/cli.py:tools_register  score=0.391
    def tools_register(parser):
        subparsers = parser.add_subparsers(
            title="Tools", help="Convenience client side tools"
        )
    
        def help(args):
            parser.print_help()
    
        parser.set_defaults(func=help)
    
        sub = subparsers.add_parser("fine_tunes.prepare_data")
        sub.add_argument(
            "-f",
            "--file",
            required=True,
            help="JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed."
            "This should be the local file path.",
        )
        sub.add_argument(
            "-q",
    ----------------------------------------------------------------------


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb


key: _node_content
string_value:
{"id_": "0acb4008-3239-4b89-a8c8-a716ef862232", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b85c1bd9-db84-49d3-96c4-8df47b26d806", "node_type": "1", "metadata": {}, "hash": "eb3e2b6df99bc6e6a26ace6c032b31afe81794e5040149be68e856459919748e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3265, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Named Entity Recognition (NER) to Enrich Text

`Named Entity Recognition` (NER) is a `Natural Language Processing` task that identifies and classifies named entities (NE) into predefined semantic categories (such as persons, organizations, locations, events, time expressions, and quantities). By converting raw text into structured information, NER makes data more actionable, facilitating tasks like information extraction, data aggregation, analytics, and social media monitoring.

This notebook demonstrates how to carry out NER with [chat completion](https://platform.openai.com/docs/api-reference/chat) and [functions-calling](https://platform.openai.com/docs/guides/gpt/function-calling) to enrich a text with links to a knowledge base such as Wikipedia:

**Text:**

*In Germany, in 1440, goldsmith Johannes Gutenberg invented the movable-type printing press. His work led to an information revolution and the unprecedented mass-spread of literature throughout Europe. Modelled on the design of the existing screw presses, a single Renaissance movable-type printing press could produce up to 3,600 pages per workday.*

**Text enriched with Wikipedia links:**

*In [Germany](https://en.wikipedia.org/wiki/Germany), in 1440, goldsmith [Johannes Gutenberg]() invented the [movable-type printing press](https://en.wikipedia.org/wiki/Movable_Type). His work led to an [information revolution](https://en.wikipedia.org/wiki/Information_revolution) and the unprecedented mass-spread of literature throughout [Europe](https://en.wikipedia.org/wiki/Europe). Modelled on the design of the existing screw presses, a single [Renaissance](https://en.wikipedia.org/wiki/Renaissance) [movable-type printing press](https://en.wikipedia.org/wiki/Movable_Type) could produce up to 3,600 pages per workday.*

**Inference Costs:** The notebook also illustrates how to estimate OpenAI API costs.

### 1. Setup

#### 1.1 Install/Upgrade Python packages


```python
%pip install --upgrade openai --quiet
%pip install --upgrade nlpia2-wikipedia --quiet
%pip install --upgrade tenacity --quiet
```

    Note: you may need to restart the kernel to use updated packages.
    Note: you may need to restart the kernel to use updated packages.
    Note: you may need to restart the kernel to use updated packages.


#### 1.2 Load packages and OPENAI_API_KEY

You can generate an API key in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.

This notebook works with the latest OpeanAI models `gpt-3.5-turbo-0613` and `gpt-4-0613`.


```python
import json
import logging
import os

import openai
import wikipedia

from typing import Optional
from IPython.display import display, Markdown
from tenacity import retry, wait_random_exponential, stop_after_attempt

logging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')

OPENAI_MODEL = 'gpt-3.5-turbo-0613'

client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
```

### 2. Define the NER labels to be Identified

We define a standard set of NER labels to showcase a wide range of use cases. However, for our specific task of enriching text with knowledge base links, only a subset is practically required.


key: _node_content
string_value:
{"id_": "b85c1bd9-db84-49d3-96c4-8df47b26d806", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0acb4008-3239-4b89-a8c8-a716ef862232", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "2760bb3dd3b350ed405caea5758dcbaed250b640a4b6ccc110b79f44646fa697", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "87698f77-bc26-4b5d-9cf1-828454540c2c", "node_type": "1", "metadata": {}, "hash": "4126aa6cf3da163ac65f4fe76981ffa7e1fbf9a10ddb43bd9d67518b61748c1b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1943, "end_char_idx": 5220, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
%pip install --upgrade openai --quiet
%pip install --upgrade nlpia2-wikipedia --quiet
%pip install --upgrade tenacity --quiet
```

    Note: you may need to restart the kernel to use updated packages.
    Note: you may need to restart the kernel to use updated packages.
    Note: you may need to restart the kernel to use updated packages.


#### 1.2 Load packages and OPENAI_API_KEY

You can generate an API key in the OpenAI web interface. See https://platform.openai.com/account/api-keys for details.

This notebook works with the latest OpeanAI models `gpt-3.5-turbo-0613` and `gpt-4-0613`.


```python
import json
import logging
import os

import openai
import wikipedia

from typing import Optional
from IPython.display import display, Markdown
from tenacity import retry, wait_random_exponential, stop_after_attempt

logging.basicConfig(level=logging.INFO, format=' %(asctime)s - %(levelname)s - %(message)s')

OPENAI_MODEL = 'gpt-3.5-turbo-0613'

client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
```

### 2. Define the NER labels to be Identified

We define a standard set of NER labels to showcase a wide range of use cases. However, for our specific task of enriching text with knowledge base links, only a subset is practically required.


```python
labels = [
    "person",      # people, including fictional characters
    "fac",         # buildings, airports, highways, bridges
    "org",         # organizations, companies, agencies, institutions
    "gpe",         # geopolitical entities like countries, cities, states
    "loc",         # non-gpe locations
    "product",     # vehicles, foods, appareal, appliances, software, toys 
    "event",       # named sports, scientific milestones, historical events
    "work_of_art", # titles of books, songs, movies
    "law",         # named laws, acts, or legislations
    "language",    # any named language
    "date",        # absolute or relative dates or periods
    "time",        # time units smaller than a day
    "percent",     # percentage (e.g., "twenty percent", "18%")
    "money",       # monetary values, including unit
    "quantity",    # measurements, e.g., weight or distance
]
```

### 3. Prepare messages

The [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) takes a list of messages as input and delivers a model-generated message as an output. While the chat format is primarily designed for facilitating multi-turn conversations, it is equally efficient for single-turn tasks without any preceding conversation. For our purposes, we will specify a message for the system, assistant, and user roles.

#### 3.1 System Message

The `system message` (prompt) sets the assistant's behavior by defining its desired persona and task. We also delineate the specific set of entity labels we aim to identify.

Although one can instruct the model to format its response, it has to be noted that both `gpt-3.5-turbo-0613` and `gpt-4-0613` have been fine-tuned to discern when a function should be invoked, and to reply with `JSON` formatted according to the function's signature. This capability streamlines our prompt and enables us to receive structured data directly from the model.


key: _node_content
string_value:
{"id_": "87698f77-bc26-4b5d-9cf1-828454540c2c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b85c1bd9-db84-49d3-96c4-8df47b26d806", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "105c2fbf95661a24dbe22fd64d0a2639b96fd5c11aeb7909d86f9a9330408a6a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "07144573-269a-44be-8a75-b7eab6da8331", "node_type": "1", "metadata": {}, "hash": "36f625d5bbeadd68399cc4a41637eea392c7e93387bb5761832d3d0f9a703bf0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5223, "end_char_idx": 8183, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def system_message(labels):
    return f"""
You are an expert in Natural Language Processing. Your task is to identify common Named Entities (NER) in a given text.
The possible common Named Entities (NER) types are exclusively: ({", ".join(labels)})."""
```

#### 3.2 Assistant Message

`Assistant messages` usually store previous assistant responses. However, as in our scenario, they can also be crafted to provide examples of the desired behavior. While OpenAI is able to execute `zero-shot` Named Entity Recognition, we have found that a `one-shot` approach produces more precise results.


```python
def assisstant_message():
    return f"""
EXAMPLE:
    Text: 'In Germany, in 1440, goldsmith Johannes Gutenberg invented the movable-type printing press. His work led to an information revolution and the unprecedented mass-spread / 
    of literature throughout Europe. Modelled on the design of the existing screw presses, a single Renaissance movable-type printing press could produce up to 3,600 pages per workday.'
    {{
        "gpe": ["Germany", "Europe"],
        "date": ["1440"],
        "person": ["Johannes Gutenberg"],
        "product": ["movable-type printing press"],
        "event": ["Renaissance"],
        "quantity": ["3,600 pages"],
        "time": ["workday"]
    }}
--"""
```

#### 3.3 User Message

The `user message` provides the specific text for the assistant task:


```python
def user_message(text):
    return f"""
TASK:
    Text: {text}
"""
```

### 4. OpenAI Functions (and Utils)

In an OpenAI API call, we can describe `functions` to `gpt-3.5-turbo-0613` and `gpt-4-0613` and have the model intelligently choose to output a `JSON` object containing arguments to call those `functions`. It's important to note that the [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) doesn't actually execute the `function`. Instead, it provides the `JSON` output, which can then be used to call the `function` in our code. For more details, refer to the [OpenAI Function Calling Guide](https://platform.openai.com/docs/guides/function-calling).

Our function, `enrich_entities(text, label_entities)` gets a block of text and a dictionary containing identified labels and entities as parameters. It then associates the recognized entities with their corresponding links to the Wikipedia articles.


```python
@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))
def find_link(entity: str) -> Optional[str]:
    """
    Finds a Wikipedia link for a given entity.
    """
    try:
        titles = wikipedia.search(entity)
        if titles:
            # naively consider the first result as the best
            page = wikipedia.page(titles[0])
            return page.url
    except (wikipedia.exceptions.WikipediaException) as ex:
        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')

    return None
```


key: _node_content
string_value:
{"id_": "07144573-269a-44be-8a75-b7eab6da8331", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "87698f77-bc26-4b5d-9cf1-828454540c2c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "849af42ad5b2866346a11f8e19d4f5a3b115495153671468e226f1beae21bfae", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7dbb4709-0527-4f6f-a161-18cb141f55e8", "node_type": "1", "metadata": {}, "hash": "7660c9d6abb17a41374d6eba5ce131692ac72e9cc6b32a48df624513bf3a5e99", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6634, "end_char_idx": 8605, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def user_message(text):
    return f"""
TASK:
    Text: {text}
"""
```

### 4. OpenAI Functions (and Utils)

In an OpenAI API call, we can describe `functions` to `gpt-3.5-turbo-0613` and `gpt-4-0613` and have the model intelligently choose to output a `JSON` object containing arguments to call those `functions`. It's important to note that the [chat completions API](https://platform.openai.com/docs/guides/gpt/chat-completions-api) doesn't actually execute the `function`. Instead, it provides the `JSON` output, which can then be used to call the `function` in our code. For more details, refer to the [OpenAI Function Calling Guide](https://platform.openai.com/docs/guides/function-calling).

Our function, `enrich_entities(text, label_entities)` gets a block of text and a dictionary containing identified labels and entities as parameters. It then associates the recognized entities with their corresponding links to the Wikipedia articles.


```python
@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))
def find_link(entity: str) -> Optional[str]:
    """
    Finds a Wikipedia link for a given entity.
    """
    try:
        titles = wikipedia.search(entity)
        if titles:
            # naively consider the first result as the best
            page = wikipedia.page(titles[0])
            return page.url
    except (wikipedia.exceptions.WikipediaException) as ex:
        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')

    return None
```


```python
def find_all_links(label_entities:dict) -> dict:
    """ 
    Finds all Wikipedia links for the dictionary entities in the whitelist label list.
    """
    whitelist = ['event', 'gpe', 'org', 'person', 'product', 'work_of_art']
    
    return {e: find_link(e) for label, entities in label_entities.items() 
                            for e in entities
                            if label in whitelist}
```


key: _node_content
string_value:
{"id_": "7dbb4709-0527-4f6f-a161-18cb141f55e8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "07144573-269a-44be-8a75-b7eab6da8331", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "70788201670f03b5f20df3f882f3e5664a879f0807d71f4db74e90a7ca4046ab", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fbb8a2f1-b476-4797-98d2-376b91545b95", "node_type": "1", "metadata": {}, "hash": "98cd09c61cfd0ec9e2b691ae4b7af41615e139b060f18bc4a839bb995b36a977", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7595, "end_char_idx": 10365, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))
def find_link(entity: str) -> Optional[str]:
    """
    Finds a Wikipedia link for a given entity.
    """
    try:
        titles = wikipedia.search(entity)
        if titles:
            # naively consider the first result as the best
            page = wikipedia.page(titles[0])
            return page.url
    except (wikipedia.exceptions.WikipediaException) as ex:
        logging.error(f'Error occurred while searching for Wikipedia link for entity {entity}: {str(ex)}')

    return None
```


```python
def find_all_links(label_entities:dict) -> dict:
    """ 
    Finds all Wikipedia links for the dictionary entities in the whitelist label list.
    """
    whitelist = ['event', 'gpe', 'org', 'person', 'product', 'work_of_art']
    
    return {e: find_link(e) for label, entities in label_entities.items() 
                            for e in entities
                            if label in whitelist}
```


```python
def enrich_entities(text: str, label_entities: dict) -> str:
    """
    Enriches text with knowledge base links.
    """
    entity_link_dict = find_all_links(label_entities)
    logging.info(f"entity_link_dict: {entity_link_dict}")
    
    for entity, link in entity_link_dict.items():
        text = text.replace(entity, f"[{entity}]({link})")

    return text
```

### 4. ChatCompletion

As previously highlighted, `gpt-3.5-turbo-0613` and `gpt-4-0613` have been fine-tuned to detect when a `function` should to be called. Moreover, they can produce a `JSON` response that conforms to the `function` signature. Here's the sequence we follow:

1. Define our `function` and its associated `JSON` Schema.
2. Invoke the model using the `messages`, `tools` and `tool_choice` parameters.
3. Convert the output into a `JSON` object, and then call the `function` with the `arguments` provided by the model.

In practice, one might want to re-invoke the model again by appending the `function` response as a new message, and let the model summarize the results back to the user. Nevertheless, for our purposes, this step is not needed.

*Note that in a real-case scenario it is strongly recommended to build in user confirmation flows before taking actions.*

#### 4.1 Define our Function and JSON schema

Since we want the model to output a dictionary of labels and recognized entities:

```python
{   
    "gpe": ["Germany", "Europe"],   
    "date": ["1440"],   
    "person": ["Johannes Gutenberg"],   
    "product": ["movable-type printing press"],   
    "event": ["Renaissance"],   
    "quantity": ["3,600 pages"],   
    "time": ["workday"]   
}   
```
we need to define the corresponding `JSON` schema to be passed to the `tools` parameter:


key: _node_content
string_value:
{"id_": "fbb8a2f1-b476-4797-98d2-376b91545b95", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7dbb4709-0527-4f6f-a161-18cb141f55e8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "adb929e5c5784d61db7f9faf96bbbd5fc37610e4c15646aad313d9468c689006", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3617e664-073e-4d02-b405-14fe68f6a451", "node_type": "1", "metadata": {}, "hash": "fd52f3aa36c997a614dd853d9b03ce869acc18b5f884cfec05b4c1c49ab5ecc6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10369, "end_char_idx": 12927, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def generate_functions(labels: dict) -> list:
    return [
        {   
            "type": "function",
            "function": {
                "name": "enrich_entities",
                "description": "Enrich Text with Knowledge Base Links",
                "parameters": {
                    "type": "object",
                        "properties": {
                            "r'^(?:' + '|'.join({labels}) + ')$'": 
                            {
                                "type": "array",
                                "items": {
                                    "type": "string"
                                }
                            }
                        },
                        "additionalProperties": False
                },
            }
        }
    ]
```

#### 4.2 Chat Completion

Now, we invoke the model. It's important to note that we direct the API to use a specific function by setting the `tool_choice` parameter to `{"type": "function", "function" : {"name": "enrich_entities"}}`.


```python
@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))
def run_openai_task(labels, text):
    messages = [
          {"role": "system", "content": system_message(labels=labels)},
          {"role": "assistant", "content": assisstant_message()},
          {"role": "user", "content": user_message(text=text)}
      ]

    # TODO: functions and function_call are deprecated, need to be updated
    # See: https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools
    response = openai.chat.completions.create(
        model="gpt-3.5-turbo-0613",
        messages=messages,
        tools=generate_functions(labels),
        tool_choice={"type": "function", "function" : {"name": "enrich_entities"}}, 
        temperature=0,
        frequency_penalty=0,
        presence_penalty=0,
    )

    response_message = response.choices[0].message
    
    available_functions = {"enrich_entities": enrich_entities}  
    function_name = response_message.tool_calls[0].function.name
    
    function_to_call = available_functions[function_name]
    logging.info(f"function_to_call: {function_to_call}")

    function_args = json.loads(response_message.tool_calls[0].function.arguments)
    logging.info(f"function_args: {function_args}")

    function_response = function_to_call(text, function_args)

    return {"model_response": response, 
            "function_response": function_response}
```

### 5. Let's Enrich a Text with Wikipedia links

#### 5.1 Run OpenAI Task


key: _node_content
string_value:
{"id_": "3617e664-073e-4d02-b405-14fe68f6a451", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fbb8a2f1-b476-4797-98d2-376b91545b95", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "cab0d4263f7340b738188bec7e3df4bc0147f148a8bfe53f4d4b8394bc9e8356", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8d7ddbad-9bfe-41c6-ad65-61fec2588b37", "node_type": "1", "metadata": {}, "hash": "f3612cf294d02603f04246cab52e142f369399fe139adb5c8d3869dd84023926", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11411, "end_char_idx": 14027, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
@retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(5))
def run_openai_task(labels, text):
    messages = [
          {"role": "system", "content": system_message(labels=labels)},
          {"role": "assistant", "content": assisstant_message()},
          {"role": "user", "content": user_message(text=text)}
      ]

    # TODO: functions and function_call are deprecated, need to be updated
    # See: https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools
    response = openai.chat.completions.create(
        model="gpt-3.5-turbo-0613",
        messages=messages,
        tools=generate_functions(labels),
        tool_choice={"type": "function", "function" : {"name": "enrich_entities"}}, 
        temperature=0,
        frequency_penalty=0,
        presence_penalty=0,
    )

    response_message = response.choices[0].message
    
    available_functions = {"enrich_entities": enrich_entities}  
    function_name = response_message.tool_calls[0].function.name
    
    function_to_call = available_functions[function_name]
    logging.info(f"function_to_call: {function_to_call}")

    function_args = json.loads(response_message.tool_calls[0].function.arguments)
    logging.info(f"function_args: {function_args}")

    function_response = function_to_call(text, function_args)

    return {"model_response": response, 
            "function_response": function_response}
```

### 5. Let's Enrich a Text with Wikipedia links

#### 5.1 Run OpenAI Task


```python
text = """The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr."""
result = run_openai_task(labels, text)
```

     2023-10-20 18:05:51,729 - INFO - function_to_call: <function enrich_entities at 0x0000021D30C462A0>
     2023-10-20 18:05:51,730 - INFO - function_args: {'person': ['John Lennon', 'Paul McCartney', 'George Harrison', 'Ringo Starr'], 'org': ['The Beatles'], 'gpe': ['Liverpool'], 'date': ['1960']}
     2023-10-20 18:06:09,858 - INFO - entity_link_dict: {'John Lennon': 'https://en.wikipedia.org/wiki/John_Lennon', 'Paul McCartney': 'https://en.wikipedia.org/wiki/Paul_McCartney', 'George Harrison': 'https://en.wikipedia.org/wiki/George_Harrison', 'Ringo Starr': 'https://en.wikipedia.org/wiki/Ringo_Starr', 'The Beatles': 'https://en.wikipedia.org/wiki/The_Beatles', 'Liverpool': 'https://en.wikipedia.org/wiki/Liverpool'}


#### 5.2 Function Response


```python
display(Markdown(f"""**Text:** {text}   
                     **Enriched_Text:** {result['function_response']}"""))
```


key: _node_content
string_value:
{"id_": "8d7ddbad-9bfe-41c6-ad65-61fec2588b37", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "13dd6230-0b7a-49b0-aa23-5f419986978c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "629ecf3c2e6d862f646f368a18ac95c95e52a0dd04ff2d164749504c14cd5c62", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3617e664-073e-4d02-b405-14fe68f6a451", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Named_Entity_Recognition_to_enrich_text.ipynb", "title": "estimate inference cost assuming gpt-3.5-turbo (4K context)", "tokens": 3604, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "66d69687369b9348514cb98f52d1b2cc2c59ba3d7e4e274793719f566462f2ce", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12930, "end_char_idx": 15361, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
text = """The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr."""
result = run_openai_task(labels, text)
```

     2023-10-20 18:05:51,729 - INFO - function_to_call: <function enrich_entities at 0x0000021D30C462A0>
     2023-10-20 18:05:51,730 - INFO - function_args: {'person': ['John Lennon', 'Paul McCartney', 'George Harrison', 'Ringo Starr'], 'org': ['The Beatles'], 'gpe': ['Liverpool'], 'date': ['1960']}
     2023-10-20 18:06:09,858 - INFO - entity_link_dict: {'John Lennon': 'https://en.wikipedia.org/wiki/John_Lennon', 'Paul McCartney': 'https://en.wikipedia.org/wiki/Paul_McCartney', 'George Harrison': 'https://en.wikipedia.org/wiki/George_Harrison', 'Ringo Starr': 'https://en.wikipedia.org/wiki/Ringo_Starr', 'The Beatles': 'https://en.wikipedia.org/wiki/The_Beatles', 'Liverpool': 'https://en.wikipedia.org/wiki/Liverpool'}


#### 5.2 Function Response


```python
display(Markdown(f"""**Text:** {text}   
                     **Enriched_Text:** {result['function_response']}"""))
```


**Text:** The Beatles were an English rock band formed in Liverpool in 1960, comprising John Lennon, Paul McCartney, George Harrison, and Ringo Starr.   
                     **Enriched_Text:** [The Beatles](https://en.wikipedia.org/wiki/The_Beatles) were an English rock band formed in [Liverpool](https://en.wikipedia.org/wiki/Liverpool) in 1960, comprising [John Lennon](https://en.wikipedia.org/wiki/John_Lennon), [Paul McCartney](https://en.wikipedia.org/wiki/Paul_McCartney), [George Harrison](https://en.wikipedia.org/wiki/George_Harrison), and [Ringo Starr](https://en.wikipedia.org/wiki/Ringo_Starr).


#### 5.3 Token Usage

To estimate the inference costs, we can parse the response's "usage" field. Detailed token costs per model are available in the [OpenAI Pricing Guide](https://openai.com/pricing):


```python
# estimate inference cost assuming gpt-3.5-turbo (4K context)
i_tokens  = result["model_response"].usage.prompt_tokens 
o_tokens = result["model_response"].usage.completion_tokens 

i_cost = (i_tokens / 1000) * 0.0015
o_cost = (o_tokens / 1000) * 0.002

print(f"""Token Usage
    Prompt: {i_tokens} tokens
    Completion: {o_tokens} tokens
    Cost estimation: ${round(i_cost + o_cost, 5)}""")
```

    Token Usage
        Prompt: 331 tokens
        Completion: 47 tokens
        Cost estimation: $0.00059


key: _node_content
string_value:
{"id_": "3fcc1241-17d9-48d8-8246-888400b00ff8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/README.ipynb", "title": "Vector Databases", "tokens": 586, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "8555a529-4c1f-47b9-a733-69aee759e837", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/README.ipynb", "title": "Vector Databases", "tokens": 586, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "57292be6e3afabcdaa5c7271b2922fab4af57c93d1fb1a265767a40d9c0c83b5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2256, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Vector Databases

This section of the OpenAI Cookbook showcases many of the vector databases available to support your semantic search use cases.

Vector databases can be a great accompaniment for knowledge retrieval applications, which reduce hallucinations by providing the LLM with the relevant context to answer questions.

Each provider has their own named directory, with a standard notebook to introduce you to using our API with their product, and any supplementary notebooks they choose to add to showcase their functionality.

## Guides & deep dives
- [AnalyticDB](https://www.alibabacloud.com/help/en/analyticdb-for-postgresql/latest/get-started-with-analyticdb-for-postgresql)
- [Cassandra/Astra DB](https://docs.datastax.com/en/astra-serverless/docs/vector-search/qandasimsearch-quickstart.html)
- [Azure AI Search](https://learn.microsoft.com/azure/search/search-get-started-vector)
- [Azure SQL Database](https://learn.microsoft.com/azure/azure-sql/database/ai-artificial-intelligence-intelligent-applications?view=azuresql)
- [Chroma](https://docs.trychroma.com/getting-started)
- [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/knn-search.html)
- [Hologres](https://www.alibabacloud.com/help/en/hologres/latest/procedure-to-use-hologres)
- [Kusto](https://learn.microsoft.com/en-us/azure/data-explorer/web-query-data)
- [Milvus](https://milvus.io/docs/example_code.md)
- [MyScale](https://docs.myscale.com/en/quickstart/)
- [MongoDB](https://www.mongodb.com/products/platform/atlas-vector-search)
- [Neon Postgres](https://neon.tech/docs/ai/ai-intro)
- [Pinecone](https://docs.pinecone.io/docs/quickstart)
- [PolarDB](https://www.alibabacloud.com/help/en/polardb/latest/quick-start)
- [Qdrant](https://qdrant.tech/documentation/quick-start/)
- [Redis](https://github.com/RedisVentures/simple-vecsim-intro)
- [SingleStoreDB](https://www.singlestore.com/blog/how-to-get-started-with-singlestore/)
- [Supabase](https://supabase.com/docs/guides/ai)
- [Tembo](https://tembo.io/docs/product/stacks/ai/vectordb)
- [Typesense](https://typesense.org/docs/guide/)
- [Vespa AI](https://vespa.ai/)
- [Weaviate](https://weaviate.io/developers/weaviate/quickstart)
- [Zilliz](https://docs.zilliz.com/docs/quick-start-1)


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb


key: _node_content
string_value:
{"id_": "e7190cd5-797f-491c-95b9-dae4b65c71fd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7a7d7160-1636-4a30-887e-08d518422e34", "node_type": "1", "metadata": {}, "hash": "6b41a7a857a0b428bd46502f5bc5e40174f60e462e440a2d7d51cac8d7368019", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2225, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Kusto as a Vector database for AI embeddings

This Notebook provides step by step instuctions on using Azure Data Explorer (Kusto) as a vector database with OpenAI embeddings. 

This notebook presents an end-to-end process of:

1. Using precomputed embeddings created by OpenAI API.
2. Storing the embeddings in Kusto.
3. Converting raw text query to an embedding with OpenAI API.
4. Using Kusto to perform cosine similarity search in the stored embeddings


### Prerequisites

For the purposes of this exercise we need to prepare a couple of things:

1. Azure Data Explorer(Kusto) server instance. https://azure.microsoft.com/en-us/products/data-explorer
3. Azure OpenAI credentials or OpenAI API key.


```python
%pip install wget
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)







    Collecting wget
      Downloading wget-3.2.zip (10 kB)
      Preparing metadata (setup.py) ... [?25ldone
    [?25hBuilding wheels for collected packages: wget
      Building wheel for wget (setup.py) ... [?25l- done
    [?25h  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9657 sha256=10fd8aa1d20fd49c36389dc888acc721d0578c5a0635fc9fc5dc642c0f49522e
      Stored in directory: /home/trusted-service-user/.cache/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769
    Successfully built wget
    Installing collected packages: wget
    Successfully installed wget-3.2
    
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;49m -> [0m[32;49m23.1.2[0m
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip[0m
    Note: you may need to restart the kernel to use updated packages.







    Warning: PySpark kernel has been restarted to use updated packages.
    



```python
%pip install openai
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)







    Collecting openai
      Downloading openai-0.27.6-py3-none-any.whl (71 kB)
    [2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m71.9/71.9 kB[0m [31m1.


key: _node_content
string_value:
{"id_": "7a7d7160-1636-4a30-887e-08d518422e34", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e7190cd5-797f-491c-95b9-dae4b65c71fd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4c758903eaa6bea9a98c41dbe8f1376d023f0199aa6b3a0ceb6779a6d4510f98", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fe4349ab-f091-4038-8c3d-b6ef616838c0", "node_type": "1", "metadata": {}, "hash": "9d8556a7c65a4f971d45585bf6ec0c76dbc138fe7cdd2bcc95d467074c37db4e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1852, "end_char_idx": 4267, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Warning: PySpark kernel has been restarted to use updated packages.
    



```python
%pip install openai
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)







    Collecting openai
      Downloading openai-0.27.6-py3-none-any.whl (71 kB)
    [2K     [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m71.9/71.9 kB[0m [31m1.7 MB/s[0m eta [36m0:00:00[0m00:01[0m
    [?25hRequirement already satisfied: tqdm in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (4.65.0)
    Requirement already satisfied: requests>=2.20 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (2.28.2)
    Requirement already satisfied: aiohttp in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from openai) (3.8.4)
    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (1.26.14)
    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)
    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)
    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2.1.1)
    Requirement already satisfied: attrs>=17.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (22.2.0)
    Requirement already satisfied: frozenlist>=1.1.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)
    Requirement already satisfied: multidict<7.0,>=4.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)
    Requirement already satisfied: yarl<2.0,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.8.2)
    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (4.0.


key: _node_content
string_value:
{"id_": "fe4349ab-f091-4038-8c3d-b6ef616838c0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7a7d7160-1636-4a30-887e-08d518422e34", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3615c9a61e90e315000ccc7dc54f53823a4c9d13d80f211695285849aea42513", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2b3fc3d6-c969-463f-8c34-49c4e9791983", "node_type": "1", "metadata": {}, "hash": "13e62d288df42ea07bc9615c59b32325d0cc5c6e0da7004563f384634ae316c4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2951, "end_char_idx": 5274, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2022.12.7)
    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (3.4)
    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.20->openai) (2.1.1)
    Requirement already satisfied: attrs>=17.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (22.2.0)
    Requirement already satisfied: frozenlist>=1.1.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.3)
    Requirement already satisfied: multidict<7.0,>=4.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (6.0.4)
    Requirement already satisfied: yarl<2.0,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.8.2)
    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (4.0.2)
    Requirement already satisfied: aiosignal>=1.1.2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)
    Installing collected packages: openai
    Successfully installed openai-0.27.6
    
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;49m -> [0m[32;49m23.1.2[0m
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip[0m
    Note: you may need to restart the kernel to use updated packages.







    Warning: PySpark kernel has been restarted to use updated packages.
    



```python
%pip install azure-kusto-data
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)







    Requirement already satisfied: azure-kusto-data in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (4.


key: _node_content
string_value:
{"id_": "2b3fc3d6-c969-463f-8c34-49c4e9791983", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fe4349ab-f091-4038-8c3d-b6ef616838c0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "12f042a43aaf529f56d9d41b268f5e808b3890b14ee96288462312eb3ce801d1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dc5f0f09-1d9d-495f-b376-55c3477a2c1b", "node_type": "1", "metadata": {}, "hash": "d263f5831585e4b0b36607b88507402a8388837623bc17963ce9d8db7da5a0de", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4222, "end_char_idx": 6483, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
10/site-packages (from aiohttp->openai) (4.0.2)
    Requirement already satisfied: aiosignal>=1.1.2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from aiohttp->openai) (1.3.1)
    Installing collected packages: openai
    Successfully installed openai-0.27.6
    
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;49m -> [0m[32;49m23.1.2[0m
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip[0m
    Note: you may need to restart the kernel to use updated packages.







    Warning: PySpark kernel has been restarted to use updated packages.
    



```python
%pip install azure-kusto-data
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, -1, Finished, Available)







    Requirement already satisfied: azure-kusto-data in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (4.1.4)
    Requirement already satisfied: msal<2,>=1.9.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.21.0)
    Requirement already satisfied: python-dateutil>=2.8.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.8.2)
    Requirement already satisfied: azure-core<2,>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.26.4)
    Requirement already satisfied: requests>=2.13.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.28.2)
    Requirement already satisfied: ijson~=3.1 in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (from azure-kusto-data) (3.2.0.post0)
    Requirement already satisfied: azure-identity<2,>=1.5.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.12.0)
    Requirement already satisfied: six>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (1.16.


key: _node_content
string_value:
{"id_": "dc5f0f09-1d9d-495f-b376-55c3477a2c1b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2b3fc3d6-c969-463f-8c34-49c4e9791983", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "6a806adf8e85e8276f14c8725ef14d7350ed0a7285c01b3b2e6704219225f50d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1af47952-00b2-43cb-83bc-f562ce7f0a2c", "node_type": "1", "metadata": {}, "hash": "43b6a42f3a930bf9a8feb2b0cf23aaa3b919c99fd96e082c8ab1861460c4bee2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5254, "end_char_idx": 7706, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
10/site-packages (4.1.4)
    Requirement already satisfied: msal<2,>=1.9.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.21.0)
    Requirement already satisfied: python-dateutil>=2.8.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.8.2)
    Requirement already satisfied: azure-core<2,>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.26.4)
    Requirement already satisfied: requests>=2.13.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (2.28.2)
    Requirement already satisfied: ijson~=3.1 in /nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/lib/python3.10/site-packages (from azure-kusto-data) (3.2.0.post0)
    Requirement already satisfied: azure-identity<2,>=1.5.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-kusto-data) (1.12.0)
    Requirement already satisfied: six>=1.11.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (1.16.0)
    Requirement already satisfied: typing-extensions>=4.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (4.5.0)
    Requirement already satisfied: cryptography>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (40.0.1)
    Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (1.0.0)
    Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal<2,>=1.9.0->azure-kusto-data) (2.6.0)
    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (1.26.14)
    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2.1.1)
    Requirement already satisfied: idna<4,>=2.


key: _node_content
string_value:
{"id_": "1af47952-00b2-43cb-83bc-f562ce7f0a2c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dc5f0f09-1d9d-495f-b376-55c3477a2c1b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ee13f766bd782735d293339d4fd671fb4819c91b9c2cc1b3ef5c25867f43f311", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "33224cf4-b00d-40f0-aef4-68d1122c6366", "node_type": "1", "metadata": {}, "hash": "56feecf0ba36b28225d69ec59cffe2f9542b12e3538bcdc653311fb429261b63", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6449, "end_char_idx": 8800, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
>=1.11.0->azure-kusto-data) (1.16.0)
    Requirement already satisfied: typing-extensions>=4.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-core<2,>=1.11.0->azure-kusto-data) (4.5.0)
    Requirement already satisfied: cryptography>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (40.0.1)
    Requirement already satisfied: msal-extensions<2.0.0,>=0.3.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from azure-identity<2,>=1.5.0->azure-kusto-data) (1.0.0)
    Requirement already satisfied: PyJWT[crypto]<3,>=1.0.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal<2,>=1.9.0->azure-kusto-data) (2.6.0)
    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (1.26.14)
    Requirement already satisfied: charset-normalizer<4,>=2 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2.1.1)
    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (3.4)
    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2022.12.7)
    Requirement already satisfied: cffi>=1.12 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (1.15.1)
    Requirement already satisfied: portalocker<3,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity<2,>=1.5.0->azure-kusto-data) (2.7.0)
    Requirement already satisfied: pycparser in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (2.21)
    
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;


key: _node_content
string_value:
{"id_": "33224cf4-b00d-40f0-aef4-68d1122c6366", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1af47952-00b2-43cb-83bc-f562ce7f0a2c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0552266ddeeebfec8cf67d40b6640fa97b5a9d5bb7791ae8c52d03dddcf04ead", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3c87a295-305b-4439-b53a-86f3a23428f7", "node_type": "1", "metadata": {}, "hash": "6688e41007537baa45334a702bbc393a8dd0f916978bd1c220526aa8f9d3cc1f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7628, "end_char_idx": 9980, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
13.0->azure-kusto-data) (2.1.1)
    Requirement already satisfied: idna<4,>=2.5 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (3.4)
    Requirement already satisfied: certifi>=2017.4.17 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from requests>=2.13.0->azure-kusto-data) (2022.12.7)
    Requirement already satisfied: cffi>=1.12 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (1.15.1)
    Requirement already satisfied: portalocker<3,>=1.0 in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from msal-extensions<2.0.0,>=0.3.0->azure-identity<2,>=1.5.0->azure-kusto-data) (2.7.0)
    Requirement already satisfied: pycparser in /home/trusted-service-user/cluster-env/trident_env/lib/python3.10/site-packages (from cffi>=1.12->cryptography>=2.5->azure-identity<2,>=1.5.0->azure-kusto-data) (2.21)
    
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;49m -> [0m[32;49m23.1.2[0m
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip[0m
    Note: you may need to restart the kernel to use updated packages.







    Warning: PySpark kernel has been restarted to use updated packages.
    


### Download precomputed Embeddings



In this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.



```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)





    'vector_database_wikipedia_articles_embedded.zip'




```python

import zipfile

with zipfile.ZipFile("vector_database_wikipedia_articles_embedded.zip","r") as zip_ref:
    zip_ref.extractall("/lakehouse/default/Files/data")
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)


key: _node_content
string_value:
{"id_": "3c87a295-305b-4439-b53a-86f3a23428f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "33224cf4-b00d-40f0-aef4-68d1122c6366", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f217d4500f5b19b9d46f813bc2986419fff61e289b2ab28538254d4ada43efe7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6a03e4a2-cab9-4917-8161-e09f8d96cf53", "node_type": "1", "metadata": {}, "hash": "2f58c2dc423f858fdd4c773f02d13745663c0f07b2cfe3584e1f75b04a2ec894", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8720, "end_char_idx": 10453, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
39;49m][0m[39;49m A new release of pip is available: [0m[31;49m23.0[0m[39;49m -> [0m[32;49m23.1.2[0m
    [1m[[0m[34;49mnotice[0m[1;39;49m][0m[39;49m To update, run: [0m[32;49m/nfs4/pyenv-27214bb4-edfd-4fdd-b888-8a99075a1416/bin/python -m pip install --upgrade pip[0m
    Note: you may need to restart the kernel to use updated packages.







    Warning: PySpark kernel has been restarted to use updated packages.
    


### Download precomputed Embeddings



In this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.



```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)





    'vector_database_wikipedia_articles_embedded.zip'




```python

import zipfile

with zipfile.ZipFile("vector_database_wikipedia_articles_embedded.zip","r") as zip_ref:
    zip_ref.extractall("/lakehouse/default/Files/data")
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)



```python
import pandas as pd

from ast import literal_eval

article_df = pd.read_csv('/lakehouse/default/Files/data/vector_database_wikipedia_articles_embedded.csv')
# Read vectors from strings back into a list
article_df["title_vector"] = article_df.title_vector.apply(literal_eval)
article_df["content_vector"] = article_df.content_vector.apply(literal_eval)
article_df.head()

```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)


key: _node_content
string_value:
{"id_": "6a03e4a2-cab9-4917-8161-e09f8d96cf53", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3c87a295-305b-4439-b53a-86f3a23428f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b4bb1edffeaea08416745eb18034970b9b18dae51d4642de4f4e9c247b681de4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a0929a5f-c7f9-4b6e-a890-798572598620", "node_type": "1", "metadata": {}, "hash": "420c1c7a1a4a1c15a3e8a9dd3de40c2324ff4c2b934628f0ef2e251689866033", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9087, "end_char_idx": 11470, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Warning: PySpark kernel has been restarted to use updated packages.
    


### Download precomputed Embeddings



In this section we are going to load prepared embedding data, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.



```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 17, Finished, Available)





    'vector_database_wikipedia_articles_embedded.zip'




```python

import zipfile

with zipfile.ZipFile("vector_database_wikipedia_articles_embedded.zip","r") as zip_ref:
    zip_ref.extractall("/lakehouse/default/Files/data")
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 18, Finished, Available)



```python
import pandas as pd

from ast import literal_eval

article_df = pd.read_csv('/lakehouse/default/Files/data/vector_database_wikipedia_articles_embedded.csv')
# Read vectors from strings back into a list
article_df["title_vector"] = article_df.title_vector.apply(literal_eval)
article_df["content_vector"] = article_df.content_vector.apply(literal_eval)
article_df.head()

```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)





<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.)


key: _node_content
string_value:
{"id_": "a0929a5f-c7f9-4b6e-a890-798572598620", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6a03e4a2-cab9-4917-8161-e09f8d96cf53", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "4566f47ef49daba7132989f45b43eba5d01ee7a7f4cee6e28db5b600c5cd30a5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1c2dc2ea-f26d-4296-b39d-67dfd85d09f3", "node_type": "1", "metadata": {}, "hash": "c92f1909b1deb45242cd42f34e51d158a18331b00df6dd18f3d4dd75bb0332c0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10375, "end_char_idx": 12501, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 19, Finished, Available)





<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.) is the eighth month of the year ...</td>
      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>
      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>6</td>
      <td>https://simple.wikipedia.org/wiki/Art</td>
      <td>Art</td>
      <td>Art is a creative activity that expresses imag...</td>
      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>
      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>8</td>
      <td>https://simple.wikipedia.org/wiki/A</td>
      <td>A</td>
      <td>A or a is the first letter of the English alph...</td>
      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>
      <td>[0.024894846603274345, -0.022186409682035446, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>9</td>
      <td>https://simple.wikipedia.org/wiki/Air</td>
      <td>Air</td>
      <td>Air refers to the Earth's atmosphere.


key: _node_content
string_value:
{"id_": "1c2dc2ea-f26d-4296-b39d-67dfd85d09f3", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a0929a5f-c7f9-4b6e-a890-798572598620", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2c33fd2cd0444361df088b0e3e72b7a1ae8c48c5b9376b88b824988ee374a0c1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7425717a-b88b-48b1-aeda-137fb1a860ca", "node_type": "1", "metadata": {}, "hash": "2229932697c0e43811fa611df27097cc2a7f81050bd63cb3a479f71573e44b8c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11471, "end_char_idx": 13727, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
is the eighth month of the year ...</td>
      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>
      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>6</td>
      <td>https://simple.wikipedia.org/wiki/Art</td>
      <td>Art</td>
      <td>Art is a creative activity that expresses imag...</td>
      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>
      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>8</td>
      <td>https://simple.wikipedia.org/wiki/A</td>
      <td>A</td>
      <td>A or a is the first letter of the English alph...</td>
      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>
      <td>[0.024894846603274345, -0.022186409682035446, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>9</td>
      <td>https://simple.wikipedia.org/wiki/Air</td>
      <td>Air</td>
      <td>Air refers to the Earth's atmosphere. Air is a...</td>
      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>
      <td>[0.021524671465158463, 0.018522677943110466, -...</td>
      <td>4</td>
    </tr>
  </tbody>
</table>
</div>



### Store vectors in a Kusto table


Create a table & load the vectors in Kusto based on the contents in the dataframe. The spark option CreakeIfNotExists will automatically create a table if it doesn't exist



```python
# replace with your AAD Tenant ID, Kusto Cluster URI, Kusto DB name and Kusto Table
AAD_TENANT_ID = ""
KUSTO_CLUSTER =  ""
KUSTO_DATABASE = "Vector"
KUSTO_TABLE = "Wiki"
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 37, Finished, Available)



```python

kustoOptions = {"kustoCluster": KUSTO_CLUSTER, "kustoDatabase" :KUSTO_DATABASE, "kustoTable" : KUSTO_TABLE }

# Replace the auth method based on your desired authentication mechanism  - https://github.com/Azure/azure-kusto-spark/blob/master/docs/Authentication.md
access_token=mssparkutils.credentials.getToken(kustoOptions["kustoCluster"])
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 21, Finished, Available)



```python
#Pandas data frame to spark dataframe
sparkDF=spark.createDataFrame(article_df)
```


key: _node_content
string_value:
{"id_": "7425717a-b88b-48b1-aeda-137fb1a860ca", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1c2dc2ea-f26d-4296-b39d-67dfd85d09f3", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c376eb1188813532edf2b3a1a472c4135f48e126ddd6e747d3d96745406a9763", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b4276cb4-9103-4c89-93d0-01111d5fb0c6", "node_type": "1", "metadata": {}, "hash": "948ef270714793d1c74fe9a89b7cac6865d0d6bc8d3da6ab7e6bda99903bc79c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12502, "end_char_idx": 15096, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Air is a...</td>
      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>
      <td>[0.021524671465158463, 0.018522677943110466, -...</td>
      <td>4</td>
    </tr>
  </tbody>
</table>
</div>



### Store vectors in a Kusto table


Create a table & load the vectors in Kusto based on the contents in the dataframe. The spark option CreakeIfNotExists will automatically create a table if it doesn't exist



```python
# replace with your AAD Tenant ID, Kusto Cluster URI, Kusto DB name and Kusto Table
AAD_TENANT_ID = ""
KUSTO_CLUSTER =  ""
KUSTO_DATABASE = "Vector"
KUSTO_TABLE = "Wiki"
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 37, Finished, Available)



```python

kustoOptions = {"kustoCluster": KUSTO_CLUSTER, "kustoDatabase" :KUSTO_DATABASE, "kustoTable" : KUSTO_TABLE }

# Replace the auth method based on your desired authentication mechanism  - https://github.com/Azure/azure-kusto-spark/blob/master/docs/Authentication.md
access_token=mssparkutils.credentials.getToken(kustoOptions["kustoCluster"])
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 21, Finished, Available)



```python
#Pandas data frame to spark dataframe
sparkDF=spark.createDataFrame(article_df)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 22, Finished, Available)


    /opt/spark/python/lib/pyspark.zip/pyspark/sql/pandas/conversion.py:604: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.



```python
# Write data to a Kusto table
sparkDF.write. \
format("com.microsoft.kusto.spark.synapse.datasource"). \
option("kustoCluster",kustoOptions["kustoCluster"]). \
option("kustoDatabase",kustoOptions["kustoDatabase"]). \
option("kustoTable", kustoOptions["kustoTable"]). \
option("accessToken", access_token). \
option("tableCreateOptions", "CreateIfNotExist").\
mode("Append"). \
save()

```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 23, Finished, Available)


### Prepare your OpenAI API key
# 

The OpenAI API key is used for vectorization of the documents and queries. You can follow the instructions to create and retrieve your Azure OpenAI key and endpoint. https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings


Please make sure to use the `text-embedding-3-small` model. Since the precomputed embeddings were created with `text-embedding-3-small` model we also have to use it during search.



```python
import openai
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)


#### If using Azure Open AI


key: _node_content
string_value:
{"id_": "b4276cb4-9103-4c89-93d0-01111d5fb0c6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7425717a-b88b-48b1-aeda-137fb1a860ca", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9a37c87218504a3ac0943e9feeddef3524bf90196674b69520a939e7955db953", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "93742137-206c-439e-a169-90cf65882097", "node_type": "1", "metadata": {}, "hash": "c9e8e5be9bcf5fb8b27da96d341ecd7741ceaa63c9f0b91a92423093ebafaafc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13734, "end_char_idx": 16225, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 22, Finished, Available)


    /opt/spark/python/lib/pyspark.zip/pyspark/sql/pandas/conversion.py:604: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.



```python
# Write data to a Kusto table
sparkDF.write. \
format("com.microsoft.kusto.spark.synapse.datasource"). \
option("kustoCluster",kustoOptions["kustoCluster"]). \
option("kustoDatabase",kustoOptions["kustoDatabase"]). \
option("kustoTable", kustoOptions["kustoTable"]). \
option("accessToken", access_token). \
option("tableCreateOptions", "CreateIfNotExist").\
mode("Append"). \
save()

```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 23, Finished, Available)


### Prepare your OpenAI API key
# 

The OpenAI API key is used for vectorization of the documents and queries. You can follow the instructions to create and retrieve your Azure OpenAI key and endpoint. https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings


Please make sure to use the `text-embedding-3-small` model. Since the precomputed embeddings were created with `text-embedding-3-small` model we also have to use it during search.



```python
import openai
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)


#### If using Azure Open AI


```python
openai.api_version = '2022-12-01'
openai.api_base = '' # Please add your endpoint here
openai.api_type = 'azure'
openai.api_key = ''  # Please add your api key here

def embed(query):
    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
            input=query,
            deployment_id="embed", #replace with your deployment id
            chunk_size=1
    )["data"][0]["embedding"]
    return embedded_query
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 44, Finished, Available)


#### If using Open AI

Only run this cell if you plan to use Open AI for embedding


```python
openai.api_key = ""


def embed(query):
    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small",
    )["data"][0]["embedding"]
    return embedded_query
```

### Generate embedding for the search term


```python

searchedEmbedding = embed("places where you worship")
#print(searchedEmbedding)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 45, Finished, Available)


key: _node_content
string_value:
{"id_": "93742137-206c-439e-a169-90cf65882097", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b4276cb4-9103-4c89-93d0-01111d5fb0c6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "43acb5c7ecd5f2a3064f6fc222271c4d0d826d75d1bf014bf9740a28f927f85f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c60e01ef-9ef3-4476-a729-8ddc7bf2426c", "node_type": "1", "metadata": {}, "hash": "f22cfe0ac1f9f23c98bf31f5fe975b784d2af3f489297996a9332ccd171d3f8c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14954, "end_char_idx": 17577, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import openai
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 43, Finished, Available)


#### If using Azure Open AI


```python
openai.api_version = '2022-12-01'
openai.api_base = '' # Please add your endpoint here
openai.api_type = 'azure'
openai.api_key = ''  # Please add your api key here

def embed(query):
    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
            input=query,
            deployment_id="embed", #replace with your deployment id
            chunk_size=1
    )["data"][0]["embedding"]
    return embedded_query
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 44, Finished, Available)


#### If using Open AI

Only run this cell if you plan to use Open AI for embedding


```python
openai.api_key = ""


def embed(query):
    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small",
    )["data"][0]["embedding"]
    return embedded_query
```

### Generate embedding for the search term


```python

searchedEmbedding = embed("places where you worship")
#print(searchedEmbedding)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 45, Finished, Available)


#### Semantic search in Kusto 

We will search the Kusto table for the closest vectors.

We will be using the series-cosine-similarity-fl UDF for similarity search. 

Please create the function in your database before proceeding -
https://learn.microsoft.com/en-us/azure/data-explorer/kusto/functions-library/series-cosine-similarity-fl?tabs=query-defined


```python
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoServiceError
from azure.kusto.data.helpers import dataframe_from_result_table
import pandas as pd
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)



```python
KCSB = KustoConnectionStringBuilder.with_aad_device_authentication(
    KUSTO_CLUSTER)
KCSB.authority_id = AAD_TENANT_ID
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)



```python
KUSTO_CLIENT = KustoClient(KCSB)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)



```python
KUSTO_QUERY = "Wiki | extend similarity = series_cosine_similarity_fl(dynamic("+str(searchedEmbedding)+"), content_vector,1,1) | top 10 by similarity desc "

RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)


key: _node_content
string_value:
{"id_": "c60e01ef-9ef3-4476-a729-8ddc7bf2426c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "93742137-206c-439e-a169-90cf65882097", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "aace5e91cfd8d28e425f432acfd93ab13834e83049c87cc0352b895bb85eea19", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fd81df26-35c5-428a-9d68-8a9f2a72f4df", "node_type": "1", "metadata": {}, "hash": "f1b7445c92894888646041223e0b0ffcb3bb906c10bcbf4c255d6c4e61e0093c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16586, "end_char_idx": 17744, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoServiceError
from azure.kusto.data.helpers import dataframe_from_result_table
import pandas as pd
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)



```python
KCSB = KustoConnectionStringBuilder.with_aad_device_authentication(
    KUSTO_CLUSTER)
KCSB.authority_id = AAD_TENANT_ID
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)



```python
KUSTO_CLIENT = KustoClient(KCSB)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)



```python
KUSTO_QUERY = "Wiki | extend similarity = series_cosine_similarity_fl(dynamic("+str(searchedEmbedding)+"), content_vector,1,1) | top 10 by similarity desc "

RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)



```python
df = dataframe_from_result_table(RESPONSE.primary_results[0])
df
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 49, Finished, Available)


key: _node_content
string_value:
{"id_": "fd81df26-35c5-428a-9d68-8a9f2a72f4df", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c60e01ef-9ef3-4476-a729-8ddc7bf2426c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "48a09ac6c0f67289dedff65f26fbf56b0d1692301a0ed5be360cf0b1b136fbeb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5107ae41-7a66-4d8e-8a7f-9a99a9158cd2", "node_type": "1", "metadata": {}, "hash": "fdde13f20a2be24c2f3d0e8064fbfbef7b1d4f4369356c3db5e3a7d40642c503", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16586, "end_char_idx": 19642, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
from azure.kusto.data import KustoClient, KustoConnectionStringBuilder
from azure.kusto.data.exceptions import KustoServiceError
from azure.kusto.data.helpers import dataframe_from_result_table
import pandas as pd
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 35, Finished, Available)



```python
KCSB = KustoConnectionStringBuilder.with_aad_device_authentication(
    KUSTO_CLUSTER)
KCSB.authority_id = AAD_TENANT_ID
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 38, Finished, Available)



```python
KUSTO_CLIENT = KustoClient(KCSB)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 39, Finished, Available)



```python
KUSTO_QUERY = "Wiki | extend similarity = series_cosine_similarity_fl(dynamic("+str(searchedEmbedding)+"), content_vector,1,1) | top 10 by similarity desc "

RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 48, Finished, Available)



```python
df = dataframe_from_result_table(RESPONSE.primary_results[0])
df
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 49, Finished, Available)





<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
      <th>similarity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>852</td>
      <td>https://simple.wikipedia.org/wiki/Temple</td>
      <td>Temple</td>
      <td>A temple is a building where people go to prac...</td>
      <td>[-0.021837441250681877, -0.007722342386841774,...</td>
      <td>[-0.0019541378132998943, 0.007151313126087189,...</td>
      <td>413</td>
      <td>0.834495</td>
    </tr>
    <tr>
      <th>1</th>
      <td>78094</td>
      <td>https://simple.wikipedia.org/wiki/Christian%20...</td>
      <td>Christian worship</td>
      <td>In Christianity, worship has been thought as b...</td>
      <td>[0.0017675267299637198, -0.008890199474990368,...</td>
      <td>[0.020530683919787407, 0.0024345638230443, -0....</td>
      <td>20320</td>
      <td>0.832132</td>
    </tr>
    <tr>
      <th>2</th>
      <td>59154</td>
      <td>https://simple.wikipedia.org/wiki/Service%20of...</td>
      <td>Service of worship</td>
      <td>A service of worship is a religious meeting wh...</td>
      <td>[-0.007969820871949196, 0.0004240311391185969,...</td>
      <td>[0.003784010885283351, -0.0030924836173653603,...</td>
      <td>15519</td>
      <td>0.831633</td>
    </tr>
    <tr>
      <th>3</th>
      <td>51910</td>
      <td>https://simple.wikipedia.org/wiki/Worship</td>
      <td>Worship</td>
      <td>Worship is a word often used in religion.


key: _node_content
string_value:
{"id_": "5107ae41-7a66-4d8e-8a7f-9a99a9158cd2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fd81df26-35c5-428a-9d68-8a9f2a72f4df", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9b3720c8915bb4821c30b52e5b43b54cf88350aa4d9a15f4db190eba560e5031", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "265b75d6-bbd4-4e01-9869-0ac2af87db26", "node_type": "1", "metadata": {}, "hash": "4049fe6e800733c814f0f34a9039f72c9bd1b4f4dd37eeeeec45e0e6afb661fb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 19644, "end_char_idx": 21533, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
It ...</td>
      <td>[0.0036036288365721703, -0.01276545226573944, ...</td>
      <td>[0.007925753481686115, -0.0110504487529397, 0....</td>
      <td>14010</td>
      <td>0.828185</td>
    </tr>
    <tr>
      <th>4</th>
      <td>29576</td>
      <td>https://simple.wikipedia.org/wiki/Altar</td>
      <td>Altar</td>
      <td>An altar is a place, often a table, where a re...</td>
      <td>[0.007887467741966248, -0.02706138789653778, -...</td>
      <td>[0.023901859298348427, -0.031175222247838977, ...</td>
      <td>8708</td>
      <td>0.824124</td>
    </tr>
    <tr>
      <th>5</th>
      <td>92507</td>
      <td>https://simple.wikipedia.org/wiki/Shrine</td>
      <td>Shrine</td>
      <td>A shrine is a holy or sacred place with someth...</td>
      <td>[-0.011601685546338558, 0.006366696208715439, ...</td>
      <td>[0.016423320397734642, -0.0015560361789539456,...</td>
      <td>23945</td>
      <td>0.823863</td>
    </tr>
    <tr>
      <th>6</th>
      <td>815</td>
      <td>https://simple.wikipedia.org/wiki/Synagogue</td>
      <td>Synagogue</td>
      <td>A synagogue is a place where Jews meet to wors...</td>
      <td>[-0.017317570745944977, 0.0022673190105706453,...</td>
      <td>[-0.004515442531555891, 0.003739549545571208, ...</td>
      <td>398</td>
      <td>0.819942</td>
    </tr>
    <tr>
      <th>7</th>
      <td>68080</td>
      <td>https://simple.wikipedia.org/wiki/Shinto%20shrine</td>
      <td>Shinto shrine</td>
      <td>A Shinto shrine is a sacred place or site wher...</td>
      <td>[0.0035740730818361044, 0.0028098472394049168,...</td>
      <td>[0.011014971882104874, 0.00042272370774298906,...</td>
      <td>18106</td>
      <td>0.818475</td>
    </tr>
    <tr>
      <th>8</th>
      <td>57790</td>
      <td>https://simple.wikipedia.org/wiki/Chapel</td>
      <td>Chapel</td>
      <td>A chapel is a place for Christian worship.


key: _node_content
string_value:
{"id_": "265b75d6-bbd4-4e01-9869-0ac2af87db26", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5107ae41-7a66-4d8e-8a7f-9a99a9158cd2", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "70ef3f2c9bea7fa5ed8f5c3ee101ef9a52ea0bfabf354c096eb9c106b8aa0b56", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "46604e33-cc46-42b2-af1c-c720086f2a26", "node_type": "1", "metadata": {}, "hash": "e0933c6dfc9392ec545bb692e8b53e96ff82a5d58ee7b62053e6f39937bf838b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The...</td>
      <td>[-0.01371884811669588, 0.0031672674231231213, ...</td>
      <td>[0.002526090247556567, 0.02482965588569641, 0....</td>
      <td>15260</td>
      <td>0.817608</td>
    </tr>
    <tr>
      <th>9</th>
      <td>142</td>
      <td>https://simple.wikipedia.org/wiki/Church%20%28...</td>
      <td>Church (building)</td>
      <td>A church is a building that was constructed to...</td>
      <td>[0.0021336888894438744, 0.0029748091474175453,...</td>
      <td>[0.016109377145767212, 0.022908871993422508, 0...</td>
      <td>74</td>
      <td>0.812636</td>
    </tr>
  </tbody>
</table>
</div>




```python
searchedEmbedding = embed("unfortunate events in history")

```


```python
KUSTO_QUERY = "Wiki | extend similarity = series_cosine_similarity_fl(dynamic("+str(searchedEmbedding)+"), title_vector,1,1) | top 10 by similarity desc "
RESPONSE = KUSTO_CLIENT.execute(KUSTO_DATABASE, KUSTO_QUERY)

df = dataframe_from_result_table(RESPONSE.primary_results[0])
df
```


    StatementMeta(, 7e5070d2-4560-4fb8-a3a8-6a594acd58ab, 52, Finished, Available)





<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
      <th>similarity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>848</td>
      <td>https://simple.wikipedia.org/wiki/Tragedy</td>
      <td>Tragedy</td>
      <td>In theatre, a tragedy as defined by Aristotle .</td>
      <td>[-0.019502468407154083, -0.010160734876990318,</td>
      <td>[-0.012951433658599854, -0.018836138769984245,</td>
      <td>410</td>
      <td>0.851848</td>
    </tr>
    <tr>
      <th>1</th>
      <td>4469</td>
      <td>https://simple.wikipedia.org/wiki/The%20Holocaust</td>
      <td>The Holocaust</td>
      <td>The Holocaust, sometimes called The Shoah (), .


key: _node_content
string_value:
{"id_": "46604e33-cc46-42b2-af1c-c720086f2a26", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "265b75d6-bbd4-4e01-9869-0ac2af87db26", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "25a1d0559e13c018b2817a4644b9e46a8999f00f525f4428c4b27abf8866beeb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "303180bc-cbc4-4cae-b254-94fdf06150ee", "node_type": "1", "metadata": {}, "hash": "417340c4907f781f8894eb80f27dc594cb70dddf5021d72057f71716d3cb1c21", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
      <th>similarity</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>848</td>
      <td>https://simple.wikipedia.org/wiki/Tragedy</td>
      <td>Tragedy</td>
      <td>In theatre, a tragedy as defined by Aristotle .</td>
      <td>[-0.019502468407154083, -0.010160734876990318,</td>
      <td>[-0.012951433658599854, -0.018836138769984245,</td>
      <td>410</td>
      <td>0.851848</td>
    </tr>
    <tr>
      <th>1</th>
      <td>4469</td>
      <td>https://simple.wikipedia.org/wiki/The%20Holocaust</td>
      <td>The Holocaust</td>
      <td>The Holocaust, sometimes called The Shoah (), .</td>
      <td>[-0.030233195051550865, -0.024401605129241943,</td>
      <td>[-0.016398731619119644, -0.013267949223518372,</td>
      <td>1203</td>
      <td>0.847222</td>
    </tr>
    <tr>
      <th>2</th>
      <td>64216</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of historical plagues</td>
      <td>This list contains famous or well documented o.</td>
      <td>[-0.010667890310287476, -0.0003575817099772393.</td>
      <td>[-0.010863155126571655, -0.0012196656316518784.</td>
      <td>16859</td>
      <td>0.844411</td>
    </tr>
    <tr>
      <th>3</th>
      <td>4397</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of disasters</td>
      <td>This is a list of disasters, both natural and .</td>
      <td>[-0.02713736332952976, -0.005278210621327162, .</td>
      <td>[-0.023679986596107483, -0.006126823835074902,</td>
      <td>1158</td>
      <td>0.


key: _node_content
string_value:
{"id_": "303180bc-cbc4-4cae-b254-94fdf06150ee", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "46604e33-cc46-42b2-af1c-c720086f2a26", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "266fdc99df3642652eb29cc9fe3496b6d02db5feb666aaedf6271bdab9970031", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dafce2c2-734a-4e2c-8085-437e805076fc", "node_type": "1", "metadata": {}, "hash": "77c1b8171812972be76d3182c6486c2f84d822a4791d3aa429aa7714cd3bf9c1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
sometimes called The Shoah (), .</td>
      <td>[-0.030233195051550865, -0.024401605129241943,</td>
      <td>[-0.016398731619119644, -0.013267949223518372,</td>
      <td>1203</td>
      <td>0.847222</td>
    </tr>
    <tr>
      <th>2</th>
      <td>64216</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of historical plagues</td>
      <td>This list contains famous or well documented o.</td>
      <td>[-0.010667890310287476, -0.0003575817099772393.</td>
      <td>[-0.010863155126571655, -0.0012196656316518784.</td>
      <td>16859</td>
      <td>0.844411</td>
    </tr>
    <tr>
      <th>3</th>
      <td>4397</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of disasters</td>
      <td>This is a list of disasters, both natural and .</td>
      <td>[-0.02713736332952976, -0.005278210621327162, .</td>
      <td>[-0.023679986596107483, -0.006126823835074902,</td>
      <td>1158</td>
      <td>0.843063</td>
    </tr>
    <tr>
      <th>4</th>
      <td>23073</td>
      <td>https://simple.wikipedia.org/wiki/Disaster</td>
      <td>Disaster</td>
      <td>A disaster is something very not good that hap.</td>
      <td>[-0.018235962837934497, -0.020034968852996823,</td>
      <td>[-0.02504003793001175, 0.007415903266519308, 0.</td>
      <td>7251</td>
      <td>0.840334</td>
    </tr>
    <tr>
      <th>5</th>
      <td>4382</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of terrorist incidents</td>
      <td>The following is a list by date of acts and fa.</td>
      <td>[-0.03989032283425331, -0.012808636762201786, .</td>
      <td>[-0.045838188380002975, -0.01682935282588005, .</td>
      <td>1149</td>
      <td>0.836162</td>
    </tr>
    <tr>
      <th>6</th>
      <td>13528</td>
      <td>https://simple.wikipedia.org/wiki/A%20Series%2.</td>
      <td>A Series of Unfortunate Events</td>
      <td>A Series of Unfortunate Events is a series of .</td>
      <td>[0.


key: _node_content
string_value:
{"id_": "dafce2c2-734a-4e2c-8085-437e805076fc", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "303180bc-cbc4-4cae-b254-94fdf06150ee", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "94448199f2b955cfaf4caafa5739633663f35899e42d8755f0b92051e0e5b887", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9aedd650-448c-4643-a1f5-b512bd2007fa", "node_type": "1", "metadata": {}, "hash": "21d1b12e6b36bf914c92e15aa2b66237ef51bd2a79c391324eab94432268a216", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
843063</td>
    </tr>
    <tr>
      <th>4</th>
      <td>23073</td>
      <td>https://simple.wikipedia.org/wiki/Disaster</td>
      <td>Disaster</td>
      <td>A disaster is something very not good that hap.</td>
      <td>[-0.018235962837934497, -0.020034968852996823,</td>
      <td>[-0.02504003793001175, 0.007415903266519308, 0.</td>
      <td>7251</td>
      <td>0.840334</td>
    </tr>
    <tr>
      <th>5</th>
      <td>4382</td>
      <td>https://simple.wikipedia.org/wiki/List%20of%20.</td>
      <td>List of terrorist incidents</td>
      <td>The following is a list by date of acts and fa.</td>
      <td>[-0.03989032283425331, -0.012808636762201786, .</td>
      <td>[-0.045838188380002975, -0.01682935282588005, .</td>
      <td>1149</td>
      <td>0.836162</td>
    </tr>
    <tr>
      <th>6</th>
      <td>13528</td>
      <td>https://simple.wikipedia.org/wiki/A%20Series%2.</td>
      <td>A Series of Unfortunate Events</td>
      <td>A Series of Unfortunate Events is a series of .</td>
      <td>[0.0010618815431371331, -0.0267023965716362, -.</td>
      <td>[0.002801976166665554, -0.02904471382498741, -.</td>
      <td>4347</td>
      <td>0.835172</td>
    </tr>
    <tr>
      <th>7</th>
      <td>42874</td>
      <td>https://simple.wikipedia.org/wiki/History%20of.</td>
      <td>History of the world</td>
      <td>The history of the world (also called human hi.</td>
      <td>[0.0026915925554931164, -0.022206028923392296,</td>
      <td>[0.013645033352077007, -0.005165994167327881, .</td>
      <td>11672</td>
      <td>0.830243</td>
    </tr>
    <tr>
      <th>8</th>
      <td>4452</td>
      <td>https://simple.wikipedia.org/wiki/Accident</td>
      <td>Accident</td>
      <td>An accident is when something goes wrong when .</td>
      <td>[-0.004075294826179743, -0.0059883203357458115.</td>
      <td>[0.00926120299845934, 0.013705797493457794, 0.</td>
      <td>1190</td>
      <td>0.


key: _node_content
string_value:
{"id_": "9aedd650-448c-4643-a1f5-b512bd2007fa", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "47c8b8a5-7d46-497f-bfd4-768aa7f0ea49", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c37079eee286783390ecb5a97ad1b0f1a0f7dd8119cd22d743cedd85622980ae", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dafce2c2-734a-4e2c-8085-437e805076fc", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/Getting_started_with_kusto_and_openai_embeddings.ipynb", "title": "Kusto as a Vector database for AI embeddings", "tokens": 8919, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f62eaab12057c6165e8d4f6c4fad0405fd149a107422469c0fb933a838e2166f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</td>
      <td>A Series of Unfortunate Events</td>
      <td>A Series of Unfortunate Events is a series of .</td>
      <td>[0.0010618815431371331, -0.0267023965716362, -.</td>
      <td>[0.002801976166665554, -0.02904471382498741, -.</td>
      <td>4347</td>
      <td>0.835172</td>
    </tr>
    <tr>
      <th>7</th>
      <td>42874</td>
      <td>https://simple.wikipedia.org/wiki/History%20of.</td>
      <td>History of the world</td>
      <td>The history of the world (also called human hi.</td>
      <td>[0.0026915925554931164, -0.022206028923392296,</td>
      <td>[0.013645033352077007, -0.005165994167327881, .</td>
      <td>11672</td>
      <td>0.830243</td>
    </tr>
    <tr>
      <th>8</th>
      <td>4452</td>
      <td>https://simple.wikipedia.org/wiki/Accident</td>
      <td>Accident</td>
      <td>An accident is when something goes wrong when .</td>
      <td>[-0.004075294826179743, -0.0059883203357458115.</td>
      <td>[0.00926120299845934, 0.013705797493457794, 0.</td>
      <td>1190</td>
      <td>0.826898</td>
    </tr>
    <tr>
      <th>9</th>
      <td>324</td>
      <td>https://simple.wikipedia.org/wiki/History</td>
      <td>History</td>
      <td>History is the study of past events. People kn...</td>
      <td>[0.006603690329939127, -0.011856242083013058, ...</td>
      <td>[0.0048830462619662285, 0.0032003086525946856,...</td>
      <td>170</td>
      <td>0.824645</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "f4b28d5a-3314-414a-8851-30e87b61329d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/README.ipynb", "title": "Kusto as a Vector database", "tokens": 363, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "6beb4da0-3ed7-4423-ae65-7abd82e3164b", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/kusto/README.ipynb", "title": "Kusto as a Vector database", "tokens": 363, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "9ec102c8125dac468f4c00c162caee35feb43a3a6df9ee5811044be86f2e561a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 1681, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Kusto as a Vector database



[Azure Data Explorer aka Kusto](https://azure.microsoft.com/en-us/products/data-explorer) is a cloud-based data analytics service that enables users to perform advanced analytics on large datasets in real-time. It is particularly well-suited for handling large volumes of data, making it an excellent choice for storing and searching vectors.

Kusto supports a special data type called dynamic, which can store unstructured data such as arrays and properties bag. [Dynamic data type](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/scalar-data-types/dynamic) is perfect for storing vector values. You can further augment the vector value by storing metadata related to the original object as separate columns in your table.  
Kusto also supports in-built function [series_cosine_similarity_fl](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/functions-library/series-cosine-similarity-fl) to perform vector similarity searches.

[Get started](https://aka.ms/kustofree) with Kusto for free. 

![Kusto_Vector](./images/kusto_vector_db.png)



## Getting started with Kusto and Open AI embedding

### Demo Scenario

![Wiki_embeddings](./images/wiki_embeddings.png)

![semantic_search_flow](./images/semantic_search_user_flow.png)

If you’d like to try this demo, please follow the instructions in the [Notebook](Getting_started_with_kusto_and_openai_embeddings.ipynb).

It will allow you to -  

1. Use precomputed embeddings created by OpenAI API. 

2. Store the embeddings in Kusto. 

3. Convert raw text query to an embedding with OpenAI API. 

4. Use Kusto to perform cosine similarity search in the stored embeddings.


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb


key: _node_content
string_value:
{"id_": "141c2bf0-45f4-4e40-97fd-a6c73bf13e5d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "117045dd-8d4b-46cd-87ab-a6a0d3a086f0", "node_type": "1", "metadata": {}, "hash": "2236303d6b8edacf79adf5cde2a465109b1bc6566bf95b6a89f9bc36006f737f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3307, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Using Hologres as a vector database for OpenAI embeddings

This notebook guides you step by step on using Hologres as a vector database for OpenAI embeddings.

This notebook presents an end-to-end process of:
1. Using precomputed embeddings created by OpenAI API.
2. Storing the embeddings in a cloud instance of Hologres.
3. Converting raw text query to an embedding with OpenAI API.
4. Using Hologres to perform the nearest neighbour search in the created collection.
5. Provide large language models with the search results as context in prompt engineering

### What is Hologres

[Hologres](https://www.alibabacloud.com/help/en/hologres/latest/what-is-hologres) is a unified real-time data warehousing service developed by Alibaba Cloud. You can use Hologres to write, update, process, and analyze large amounts of data in real time. Hologres supports standard SQL syntax, is compatible with PostgreSQL, and supports most PostgreSQL functions. Hologres supports online analytical processing (OLAP) and ad hoc analysis for up to petabytes of data, and provides high-concurrency and low-latency online data services. Hologres supports fine-grained isolation of multiple workloads and enterprise-level security capabilities. Hologres is deeply integrated with MaxCompute, Realtime Compute for Apache Flink, and DataWorks, and provides full-stack online and offline data warehousing solutions for enterprises.

Hologres provides vector database functionality by adopting [Proxima](https://www.alibabacloud.com/help/en/hologres/latest/vector-processing).

Proxima is a high-performance software library developed by Alibaba DAMO Academy. It allows you to search for the nearest neighbors of vectors. Proxima provides higher stability and performance than similar open source software such as Facebook AI Similarity Search (Faiss). Proxima provides basic modules that have leading performance and effects in the industry and allows you to search for similar images, videos, or human faces. Hologres is deeply integrated with Proxima to provide a high-performance vector search service.

### Deployment options

- [Click here](https://www.alibabacloud.com/product/hologres) to fast deploy [Hologres data warehouse](https://www.alibabacloud.com/help/en/hologres/latest/getting-started).


## Prerequisites

For the purposes of this exercise we need to prepare a couple of things:

1. Hologres cloud server instance.
2. The 'psycopg2-binary' library to interact with the vector database. Any other postgresql client library is ok.
3. An [OpenAI API key](https://beta.openai.com/account/api-keys).



We might validate if the server was launched successfully by running a simple curl command:


### Install requirements

This notebook obviously requires the `openai` and `psycopg2-binary` packages, but there are also some other additional libraries we will use. The following command installs them all:



```python
! pip install openai psycopg2-binary pandas wget
```

### Prepare your OpenAI API key

The OpenAI API key is used for vectorization of the documents and queries.

If you don't have an OpenAI API key, you can get one from [https://beta.openai.com/account/api-keys](https://beta.openai.com/account/api-keys).

Once you get your key, please add it to your environment variables as `OPENAI_API_KEY`.


key: _node_content
string_value:
{"id_": "117045dd-8d4b-46cd-87ab-a6a0d3a086f0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "141c2bf0-45f4-4e40-97fd-a6c73bf13e5d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "638add20a1592838a5a10086d7d548e51f22c5b15a3edcc7c948d4d37c17669e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f1496051-7ba9-445f-8fb3-d244fceb27b9", "node_type": "1", "metadata": {}, "hash": "c4105f6c713872c0885ec57f4cd51584f75041811d3e2871f13eb9ba7cfc715e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2286, "end_char_idx": 5357, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Prerequisites

For the purposes of this exercise we need to prepare a couple of things:

1. Hologres cloud server instance.
2. The 'psycopg2-binary' library to interact with the vector database. Any other postgresql client library is ok.
3. An [OpenAI API key](https://beta.openai.com/account/api-keys).



We might validate if the server was launched successfully by running a simple curl command:


### Install requirements

This notebook obviously requires the `openai` and `psycopg2-binary` packages, but there are also some other additional libraries we will use. The following command installs them all:



```python
! pip install openai psycopg2-binary pandas wget
```

### Prepare your OpenAI API key

The OpenAI API key is used for vectorization of the documents and queries.

If you don't have an OpenAI API key, you can get one from [https://beta.openai.com/account/api-keys](https://beta.openai.com/account/api-keys).

Once you get your key, please add it to your environment variables as `OPENAI_API_KEY`.


```python
# Test that your OpenAI API key is correctly set as an environment variable
# Note. if you run this notebook locally, you will need to reload your terminal and the notebook for the env variables to be live.
import os

# Note. alternatively you can set a temporary env variable like this:
# os.environ["OPENAI_API_KEY"] = "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

if os.getenv("OPENAI_API_KEY") is not None:
    print("OPENAI_API_KEY is ready")
else:
    print("OPENAI_API_KEY environment variable not found")
```

    OPENAI_API_KEY is ready


## Connect to Hologres
First add it to your environment variables. or you can just change the "psycopg2.connect" parameters below

Connecting to a running instance of Hologres server is easy with the official Python library:


```python
import os
import psycopg2

# Note. alternatively you can set a temporary env variable like this:
# os.environ["PGHOST"] = "your_host"
# os.environ["PGPORT"] "5432"),
# os.environ["PGDATABASE"] "postgres"),
# os.environ["PGUSER"] "user"),
# os.environ["PGPASSWORD"] "password"),

connection = psycopg2.connect(
    host=os.environ.get("PGHOST", "localhost"),
    port=os.environ.get("PGPORT", "5432"),
    database=os.environ.get("PGDATABASE", "postgres"),
    user=os.environ.get("PGUSER", "user"),
    password=os.environ.get("PGPASSWORD", "password")
)
connection.set_session(autocommit=True)

# Create a new cursor object
cursor = connection.cursor()
```

We can test the connection by running any available method:


```python

# Execute a simple query to test the connection
cursor.execute("SELECT 1;")
result = cursor.fetchone()

# Check the query result
if result == (1,):
    print("Connection successful!")
else:
    print("Connection failed.")
```

    Connection successful!



```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```

The downloaded file has to be then extracted:


key: _node_content
string_value:
{"id_": "f1496051-7ba9-445f-8fb3-d244fceb27b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "117045dd-8d4b-46cd-87ab-a6a0d3a086f0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "2625ee81c2020a1493840169f3cb2b709d0a13a4bd72d17e0cba7de92f5b52d7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1067453b-810c-49e6-bbe6-c1d856889c30", "node_type": "1", "metadata": {}, "hash": "66d9df8f734b22fb4b9d9e2088f6f8c0736f964dd981da6a018711f2c379a2ae", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3869, "end_char_idx": 6823, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Connect to Hologres
First add it to your environment variables. or you can just change the "psycopg2.connect" parameters below

Connecting to a running instance of Hologres server is easy with the official Python library:


```python
import os
import psycopg2

# Note. alternatively you can set a temporary env variable like this:
# os.environ["PGHOST"] = "your_host"
# os.environ["PGPORT"] "5432"),
# os.environ["PGDATABASE"] "postgres"),
# os.environ["PGUSER"] "user"),
# os.environ["PGPASSWORD"] "password"),

connection = psycopg2.connect(
    host=os.environ.get("PGHOST", "localhost"),
    port=os.environ.get("PGPORT", "5432"),
    database=os.environ.get("PGDATABASE", "postgres"),
    user=os.environ.get("PGUSER", "user"),
    password=os.environ.get("PGPASSWORD", "password")
)
connection.set_session(autocommit=True)

# Create a new cursor object
cursor = connection.cursor()
```

We can test the connection by running any available method:


```python

# Execute a simple query to test the connection
cursor.execute("SELECT 1;")
result = cursor.fetchone()

# Check the query result
if result == (1,):
    print("Connection successful!")
else:
    print("Connection failed.")
```

    Connection successful!



```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```

The downloaded file has to be then extracted:


```python
import zipfile
import os
import re
import tempfile

current_directory = os.getcwd()
zip_file_path = os.path.join(current_directory, "vector_database_wikipedia_articles_embedded.zip")
output_directory = os.path.join(current_directory, "../../data")

with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
    zip_ref.extractall(output_directory)


# check the csv file exist
file_name = "vector_database_wikipedia_articles_embedded.csv"
data_directory = os.path.join(current_directory, "../../data")
file_path = os.path.join(data_directory, file_name)


if os.path.exists(file_path):
    print(f"The file {file_name} exists in the data directory.")
else:
    print(f"The file {file_name} does not exist in the data directory.")

```

    The file vector_database_wikipedia_articles_embedded.csv exists in the data directory.


## Load data

In this section we are going to load the data prepared previous to this session, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.


```python
!unzip -n vector_database_wikipedia_articles_embedded.zip
!ls -lh vector_database_wikipedia_articles_embedded.csv
```

    Archive:  vector_database_wikipedia_articles_embedded.zip
    -rw-r--r--@ 1 geng  staff   1.7G Jan 31 01:19 vector_database_wikipedia_articles_embedded.csv


Take a look at the data.


```python
import pandas, json
data = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')
data
```


key: _node_content
string_value:
{"id_": "1067453b-810c-49e6-bbe6-c1d856889c30", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f1496051-7ba9-445f-8fb3-d244fceb27b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "56ced9361d6401f6762c2b895c88b3e6929d2152ee6edc7caf79fb66348093fd", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0ef99c20-21d7-4e02-a37b-d29ec8681a81", "node_type": "1", "metadata": {}, "hash": "dcb5fdf35242faba40e9d7263ad8f6db55a2fd6b0f845c50e155f89f47c218fe", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5095, "end_char_idx": 7839, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import wget

embeddings_url = "https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip"

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```

The downloaded file has to be then extracted:


```python
import zipfile
import os
import re
import tempfile

current_directory = os.getcwd()
zip_file_path = os.path.join(current_directory, "vector_database_wikipedia_articles_embedded.zip")
output_directory = os.path.join(current_directory, "../../data")

with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
    zip_ref.extractall(output_directory)


# check the csv file exist
file_name = "vector_database_wikipedia_articles_embedded.csv"
data_directory = os.path.join(current_directory, "../../data")
file_path = os.path.join(data_directory, file_name)


if os.path.exists(file_path):
    print(f"The file {file_name} exists in the data directory.")
else:
    print(f"The file {file_name} does not exist in the data directory.")

```

    The file vector_database_wikipedia_articles_embedded.csv exists in the data directory.


## Load data

In this section we are going to load the data prepared previous to this session, so you don't have to recompute the embeddings of Wikipedia articles with your own credits.


```python
!unzip -n vector_database_wikipedia_articles_embedded.zip
!ls -lh vector_database_wikipedia_articles_embedded.csv
```

    Archive:  vector_database_wikipedia_articles_embedded.zip
    -rw-r--r--@ 1 geng  staff   1.7G Jan 31 01:19 vector_database_wikipedia_articles_embedded.csv


Take a look at the data.


```python
import pandas, json
data = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')
data
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.)


key: _node_content
string_value:
{"id_": "0ef99c20-21d7-4e02-a37b-d29ec8681a81", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1067453b-810c-49e6-bbe6-c1d856889c30", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c71d0254c86be16a95be8d4eaa32a68f2b4ccf27caf54a336d731fdc3c6f4d58", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a184ba02-d23c-48dc-9ef2-94f292cbdd40", "node_type": "1", "metadata": {}, "hash": "011b0d32711c90c16319018a035728a3c0ca2a04dfb1efed79bd335d8a6f47c0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6673, "end_char_idx": 8870, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Take a look at the data.


```python
import pandas, json
data = pandas.read_csv('../../data/vector_database_wikipedia_articles_embedded.csv')
data
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.) is the eighth month of the year ...</td>
      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>
      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>6</td>
      <td>https://simple.wikipedia.org/wiki/Art</td>
      <td>Art</td>
      <td>Art is a creative activity that expresses imag...</td>
      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>
      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>8</td>
      <td>https://simple.wikipedia.org/wiki/A</td>
      <td>A</td>
      <td>A or a is the first letter of the English alph...</td>
      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>
      <td>[0.024894846603274345, -0.022186409682035446, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>9</td>
      <td>https://simple.wikipedia.org/wiki/Air</td>
      <td>Air</td>
      <td>Air refers to the Earth's atmosphere.


key: _node_content
string_value:
{"id_": "a184ba02-d23c-48dc-9ef2-94f292cbdd40", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0ef99c20-21d7-4e02-a37b-d29ec8681a81", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "e817952481f58ba0aaead98384419120b6411207e9b77231f49273822ff6ce2c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dc134d6b-3fb3-4df1-bd78-81b39ed325ee", "node_type": "1", "metadata": {}, "hash": "470d5828e8cd532d739e910589b771a421cb39c440084aff18376f3ca8bd37bc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
is the eighth month of the year ...</td>
      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>
      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>6</td>
      <td>https://simple.wikipedia.org/wiki/Art</td>
      <td>Art</td>
      <td>Art is a creative activity that expresses imag...</td>
      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>
      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>8</td>
      <td>https://simple.wikipedia.org/wiki/A</td>
      <td>A</td>
      <td>A or a is the first letter of the English alph...</td>
      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>
      <td>[0.024894846603274345, -0.022186409682035446, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>9</td>
      <td>https://simple.wikipedia.org/wiki/Air</td>
      <td>Air</td>
      <td>Air refers to the Earth's atmosphere. Air is a.</td>
      <td>[0.02224554680287838, -0.02044147066771984, -0.</td>
      <td>[0.021524671465158463, 0.018522677943110466, -.</td>
      <td>4</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>24995</th>
      <td>98295</td>
      <td>https://simple.wikipedia.org/wiki/Geneva</td>
      <td>Geneva</td>
      <td>Geneva (,  ,  ,  , ) is the second biggest cit.</td>
      <td>[-0.015773078426718712, 0.01737344264984131, 0.</td>
      <td>[0.008000412955880165, 0.02008531428873539, 0.</td>
      <td>24995</td>
    </tr>
    <tr>
      <th>24996</th>
      <td>98316</td>
      <td>https://simple.wikipedia.org/wiki/Concubinage</td>
      <td>Concubinage</td>
      <td>Concubinage is the state of a woman in a relat.</td>
      <td>[-0.00519518880173564,


key: _node_content
string_value:
{"id_": "dc134d6b-3fb3-4df1-bd78-81b39ed325ee", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a184ba02-d23c-48dc-9ef2-94f292cbdd40", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "8046964a1921a60e9cdd03ea8ab33010c7555e2f9f5869909e7df0f2387120ba", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "206f11fb-58fa-4833-aef2-407df139491a", "node_type": "1", "metadata": {}, "hash": "831cab4bd3222fceba166e0d8e4406d5e7d8e5a5cd14fad71ed69eab3d0c360f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Air is a.</td>
      <td>[0.02224554680287838, -0.02044147066771984, -0.</td>
      <td>[0.021524671465158463, 0.018522677943110466, -.</td>
      <td>4</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>24995</th>
      <td>98295</td>
      <td>https://simple.wikipedia.org/wiki/Geneva</td>
      <td>Geneva</td>
      <td>Geneva (,  ,  ,  , ) is the second biggest cit.</td>
      <td>[-0.015773078426718712, 0.01737344264984131, 0.</td>
      <td>[0.008000412955880165, 0.02008531428873539, 0.</td>
      <td>24995</td>
    </tr>
    <tr>
      <th>24996</th>
      <td>98316</td>
      <td>https://simple.wikipedia.org/wiki/Concubinage</td>
      <td>Concubinage</td>
      <td>Concubinage is the state of a woman in a relat.</td>
      <td>[-0.00519518880173564, 0.005898841191083193, 0.</td>
      <td>[-0.01736736111342907, -0.002740012714639306, .</td>
      <td>24996</td>
    </tr>
    <tr>
      <th>24997</th>
      <td>98318</td>
      <td>https://simple.wikipedia.org/wiki/Mistress%20%.</td>
      <td>Mistress (lover)</td>
      <td>A mistress is a man's long term female sexual .</td>
      <td>[-0.023164259269833565, -0.02052430994808674, .</td>
      <td>[-0.017878392711281776, -0.0004517830966506153.</td>
      <td>24997</td>
    </tr>
    <tr>
      <th>24998</th>
      <td>98326</td>
      <td>https://simple.wikipedia.org/wiki/Eastern%20Front</td>
      <td>Eastern Front</td>
      <td>Eastern Front can be one of the following:\n\n.</td>
      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>
      <td>[-0.0019235472427681088, -0.004023272544145584.</td>
      <td>24998</td>
    </tr>
    <tr>
      <th>24999</th>
      <td>98327</td>
      <td>https://simple.wikipedia.org/wiki/Italian%20Ca.


key: _node_content
string_value:
{"id_": "206f11fb-58fa-4833-aef2-407df139491a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dc134d6b-3fb3-4df1-bd78-81b39ed325ee", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "cca9a0e32d42e6f392e7542bb98702c4ae613ccbe6ed270dd4da796f40f77aa4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "afe153c1-7dc8-45ce-b858-83762cc8ccc5", "node_type": "1", "metadata": {}, "hash": "65f8d3530eee2759b889e843ca6fe6dc21293706f007647d80f094916678e9a9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</td>
      <td>[-0.00519518880173564, 0.005898841191083193, 0.</td>
      <td>[-0.01736736111342907, -0.002740012714639306, .</td>
      <td>24996</td>
    </tr>
    <tr>
      <th>24997</th>
      <td>98318</td>
      <td>https://simple.wikipedia.org/wiki/Mistress%20%.</td>
      <td>Mistress (lover)</td>
      <td>A mistress is a man's long term female sexual .</td>
      <td>[-0.023164259269833565, -0.02052430994808674, .</td>
      <td>[-0.017878392711281776, -0.0004517830966506153.</td>
      <td>24997</td>
    </tr>
    <tr>
      <th>24998</th>
      <td>98326</td>
      <td>https://simple.wikipedia.org/wiki/Eastern%20Front</td>
      <td>Eastern Front</td>
      <td>Eastern Front can be one of the following:\n\n.</td>
      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>
      <td>[-0.0019235472427681088, -0.004023272544145584.</td>
      <td>24998</td>
    </tr>
    <tr>
      <th>24999</th>
      <td>98327</td>
      <td>https://simple.wikipedia.org/wiki/Italian%20Ca.</td>
      <td>Italian Campaign</td>
      <td>Italian Campaign can mean the following:\n\nTh.</td>
      <td>[-0.014151256531476974, -0.008553029969334602,</td>
      <td>[-0.011758845299482346, -0.01346028596162796, .</td>
      <td>24999</td>
    </tr>
  </tbody>
</table>
<p>25000 rows × 7 columns</p>
</div>




```python
title_vector_length = len(json.loads(data['title_vector'].iloc[0]))
content_vector_length = len(json.loads(data['content_vector'].iloc[0]))

print(title_vector_length, content_vector_length)
```

    1536 1536


### Create table and proxima vector index

Hologres stores data in __tables__ where each object is described by at least one vector. Our table will be called **articles** and each object will be described by both **title** and **content** vectors.

We will start with creating a table and create proxima indexes on both **title** and **content**, and then we will fill it with our precomputed embeddings.


key: _node_content
string_value:
{"id_": "afe153c1-7dc8-45ce-b858-83762cc8ccc5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "206f11fb-58fa-4833-aef2-407df139491a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "8136a08c6de5af07a28f349d74d6b6ed726a76d4821ebba3d613d14e9ab86843", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e12ae5f1-5701-496a-b6a6-bc3ed923aa9e", "node_type": "1", "metadata": {}, "hash": "da80087ba7bbac796d5e187be303cea52e7f10533f41f21c528b5e23684826eb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</td>
      <td>[-0.00681863259524107, 0.002171179046854377, 8.</td>
      <td>[-0.0019235472427681088, -0.004023272544145584.</td>
      <td>24998</td>
    </tr>
    <tr>
      <th>24999</th>
      <td>98327</td>
      <td>https://simple.wikipedia.org/wiki/Italian%20Ca.</td>
      <td>Italian Campaign</td>
      <td>Italian Campaign can mean the following:\n\nTh.</td>
      <td>[-0.014151256531476974, -0.008553029969334602,</td>
      <td>[-0.011758845299482346, -0.01346028596162796, .</td>
      <td>24999</td>
    </tr>
  </tbody>
</table>
<p>25000 rows × 7 columns</p>
</div>




```python
title_vector_length = len(json.loads(data['title_vector'].iloc[0]))
content_vector_length = len(json.loads(data['content_vector'].iloc[0]))

print(title_vector_length, content_vector_length)
```

    1536 1536


### Create table and proxima vector index

Hologres stores data in __tables__ where each object is described by at least one vector. Our table will be called **articles** and each object will be described by both **title** and **content** vectors.

We will start with creating a table and create proxima indexes on both **title** and **content**, and then we will fill it with our precomputed embeddings.


```python
cursor.execute('CREATE EXTENSION IF NOT EXISTS proxima;')
create_proxima_table_sql = '''
BEGIN;
DROP TABLE IF EXISTS articles;
CREATE TABLE articles (
    id INT PRIMARY KEY NOT NULL,
    url TEXT,
    title TEXT,
    content TEXT,
    title_vector float4[] check(
        array_ndims(title_vector) = 1 and 
        array_length(title_vector, 1) = 1536
    ), -- define the vectors
    content_vector float4[] check(
        array_ndims(content_vector) = 1 and 
        array_length(content_vector, 1) = 1536
    ),
    vector_id INT
);

-- Create indexes for the vector fields.
call set_table_property(
    'articles',
    'proxima_vectors', 
    '{
        "title_vector":{"algorithm":"Graph","distance_method":"Euclidean","builder_params":{"min_flush_proxima_row_count" : 10}},
        "content_vector":{"algorithm":"Graph","distance_method":"Euclidean","builder_params":{"min_flush_proxima_row_count" : 10}}
    }'
);  

COMMIT;
'''

# Execute the SQL statements (will autocommit)
cursor.execute(create_proxima_table_sql)
```

### Upload data

Now let's upload the data to the Hologres cloud instance using [COPY statement](https://www.alibabacloud.com/help/en/hologres/latest/use-the-copy-statement-to-import-or-export-data). This might take 5-10 minutes according to the network bandwidth.


key: _node_content
string_value:
{"id_": "e12ae5f1-5701-496a-b6a6-bc3ed923aa9e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "afe153c1-7dc8-45ce-b858-83762cc8ccc5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "704bc636bb0675ec790f79e2379c3d3a87bb98585baf0d81d8378bee30f97e93", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "20760963-9d2f-4096-950d-e1695519a9b9", "node_type": "1", "metadata": {}, "hash": "5fcfaaf6c49a35c048fdc4153f39b4b43acec78e7fe85381fc11b94bc2da053f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11745, "end_char_idx": 14763, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
cursor.execute('CREATE EXTENSION IF NOT EXISTS proxima;')
create_proxima_table_sql = '''
BEGIN;
DROP TABLE IF EXISTS articles;
CREATE TABLE articles (
    id INT PRIMARY KEY NOT NULL,
    url TEXT,
    title TEXT,
    content TEXT,
    title_vector float4[] check(
        array_ndims(title_vector) = 1 and 
        array_length(title_vector, 1) = 1536
    ), -- define the vectors
    content_vector float4[] check(
        array_ndims(content_vector) = 1 and 
        array_length(content_vector, 1) = 1536
    ),
    vector_id INT
);

-- Create indexes for the vector fields.
call set_table_property(
    'articles',
    'proxima_vectors', 
    '{
        "title_vector":{"algorithm":"Graph","distance_method":"Euclidean","builder_params":{"min_flush_proxima_row_count" : 10}},
        "content_vector":{"algorithm":"Graph","distance_method":"Euclidean","builder_params":{"min_flush_proxima_row_count" : 10}}
    }'
);  

COMMIT;
'''

# Execute the SQL statements (will autocommit)
cursor.execute(create_proxima_table_sql)
```

### Upload data

Now let's upload the data to the Hologres cloud instance using [COPY statement](https://www.alibabacloud.com/help/en/hologres/latest/use-the-copy-statement-to-import-or-export-data). This might take 5-10 minutes according to the network bandwidth.


```python
import io

# Path to the unzipped CSV file
csv_file_path = '../../data/vector_database_wikipedia_articles_embedded.csv'

# In SQL, arrays are surrounded by {}, rather than []
def process_file(file_path):
    with open(file_path, 'r') as file:
        for line in file:
            # Replace '[' with '{' and ']' with '}'
            modified_line = line.replace('[', '{').replace(']', '}')
            yield modified_line

# Create a StringIO object to store the modified lines
modified_lines = io.StringIO(''.join(list(process_file(csv_file_path))))

# Create the COPY command for the copy_expert method
copy_command = '''
COPY public.articles (id, url, title, content, title_vector, content_vector, vector_id)
FROM STDIN WITH (FORMAT CSV, HEADER true, DELIMITER ',');
'''

# Execute the COPY command using the copy_expert method
cursor.copy_expert(copy_command, modified_lines)
```

The proxima index will be built in the background. We can do searching during this period but the query will be slow without the vector index. Use this command to wait for finish building the index.


```python
cursor.execute('vacuum articles;')
```


```python
# Check the collection size to make sure all the points have been stored
count_sql = "select count(*) from articles;"
cursor.execute(count_sql)
result = cursor.fetchone()
print(f"Count:{result[0]}")

```

    Count:25000


## Search data

Once the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.


key: _node_content
string_value:
{"id_": "20760963-9d2f-4096-950d-e1695519a9b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e12ae5f1-5701-496a-b6a6-bc3ed923aa9e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "cf16b0e2c9b0bef2b2418a91f4b90bce4a8ba313525ca7183f45bd051e9f6793", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1b740b85-289c-418f-8cdc-f5215bac8f5d", "node_type": "1", "metadata": {}, "hash": "fce9e138c89316a953679eff10b09108029dd886ae9487c5920506bd7caab72a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13053, "end_char_idx": 15525, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import io

# Path to the unzipped CSV file
csv_file_path = '../../data/vector_database_wikipedia_articles_embedded.csv'

# In SQL, arrays are surrounded by {}, rather than []
def process_file(file_path):
    with open(file_path, 'r') as file:
        for line in file:
            # Replace '[' with '{' and ']' with '}'
            modified_line = line.replace('[', '{').replace(']', '}')
            yield modified_line

# Create a StringIO object to store the modified lines
modified_lines = io.StringIO(''.join(list(process_file(csv_file_path))))

# Create the COPY command for the copy_expert method
copy_command = '''
COPY public.articles (id, url, title, content, title_vector, content_vector, vector_id)
FROM STDIN WITH (FORMAT CSV, HEADER true, DELIMITER ',');
'''

# Execute the COPY command using the copy_expert method
cursor.copy_expert(copy_command, modified_lines)
```

The proxima index will be built in the background. We can do searching during this period but the query will be slow without the vector index. Use this command to wait for finish building the index.


```python
cursor.execute('vacuum articles;')
```


```python
# Check the collection size to make sure all the points have been stored
count_sql = "select count(*) from articles;"
cursor.execute(count_sql)
result = cursor.fetchone()
print(f"Count:{result[0]}")

```

    Count:25000


## Search data

Once the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.



```python
import openai
def query_knn(query, table_name, vector_name="title_vector", top_k=20):

    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small",
    )["data"][0]["embedding"]

    # Convert the embedded_query to PostgreSQL compatible format
    embedded_query_pg = "{" + ",".join(map(str, embedded_query)) + "}"

    # Create SQL query
    query_sql = f"""
    SELECT id, url, title, pm_approx_euclidean_distance({vector_name},'{embedded_query_pg}'::float4[]) AS distance
    FROM {table_name}
    ORDER BY distance
    LIMIT {top_k};
    """
    # Execute the query
    cursor.execute(query_sql)
    results = cursor.fetchall()

    return results
```


key: _node_content
string_value:
{"id_": "1b740b85-289c-418f-8cdc-f5215bac8f5d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "20760963-9d2f-4096-950d-e1695519a9b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "50b95cd9d07101d70376392814db3ec6481dbea47789abb9c39b02024ee9be1f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a21902d1-a625-4700-a2c4-4d512be6953e", "node_type": "1", "metadata": {}, "hash": "87ae9784febfe831317d44fdaf3b2e3b385bae5e5cb7b94ebe92c364ede354e0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14149, "end_char_idx": 16421, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
cursor.execute('vacuum articles;')
```


```python
# Check the collection size to make sure all the points have been stored
count_sql = "select count(*) from articles;"
cursor.execute(count_sql)
result = cursor.fetchone()
print(f"Count:{result[0]}")

```

    Count:25000


## Search data

Once the data is uploaded we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search. Since the precomputed embeddings were created with `text-embedding-3-small` OpenAI model we also have to use it during search.



```python
import openai
def query_knn(query, table_name, vector_name="title_vector", top_k=20):

    # Creates embedding vector from user query
    embedded_query = openai.Embedding.create(
        input=query,
        model="text-embedding-3-small",
    )["data"][0]["embedding"]

    # Convert the embedded_query to PostgreSQL compatible format
    embedded_query_pg = "{" + ",".join(map(str, embedded_query)) + "}"

    # Create SQL query
    query_sql = f"""
    SELECT id, url, title, pm_approx_euclidean_distance({vector_name},'{embedded_query_pg}'::float4[]) AS distance
    FROM {table_name}
    ORDER BY distance
    LIMIT {top_k};
    """
    # Execute the query
    cursor.execute(query_sql)
    results = cursor.fetchall()

    return results
```


```python
query_results = query_knn("modern art in Europe", "Articles")
for i, result in enumerate(query_results):
    print(f"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})")
```

    1. Museum of Modern Art (Score: 0.501)
    2. Western Europe (Score: 0.485)
    3. Renaissance art (Score: 0.479)
    4. Pop art (Score: 0.472)
    5. Northern Europe (Score: 0.461)
    6. Hellenistic art (Score: 0.458)
    7. Modernist literature (Score: 0.447)
    8. Art film (Score: 0.44)
    9. Central Europe (Score: 0.439)
    10. Art (Score: 0.437)
    11. European (Score: 0.437)
    12. Byzantine art (Score: 0.436)
    13. Postmodernism (Score: 0.435)
    14. Eastern Europe (Score: 0.433)
    15. Cubism (Score: 0.433)
    16. Europe (Score: 0.432)
    17. Impressionism (Score: 0.432)
    18. Bauhaus (Score: 0.431)
    19. Surrealism (Score: 0.429)
    20. Expressionism (Score: 0.429)


key: _node_content
string_value:
{"id_": "a21902d1-a625-4700-a2c4-4d512be6953e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "f5de0968-bddc-4432-9ff2-c75ee4d2c664", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3ebf0fc03bb8cb0cda019036db1abc158d03013b99b40bda7543918043e0a2b6", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1b740b85-289c-418f-8cdc-f5215bac8f5d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/hologres/Getting_started_with_Hologres_and_OpenAI.ipynb", "title": "Using Hologres as a vector database for OpenAI embeddings", "tokens": 4831, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3764d46578c48a12652a3f00210390d5f0082618a945b2c6a52da049ea388d96", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15528, "end_char_idx": 17388, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
query_results = query_knn("modern art in Europe", "Articles")
for i, result in enumerate(query_results):
    print(f"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})")
```

    1. Museum of Modern Art (Score: 0.501)
    2. Western Europe (Score: 0.485)
    3. Renaissance art (Score: 0.479)
    4. Pop art (Score: 0.472)
    5. Northern Europe (Score: 0.461)
    6. Hellenistic art (Score: 0.458)
    7. Modernist literature (Score: 0.447)
    8. Art film (Score: 0.44)
    9. Central Europe (Score: 0.439)
    10. Art (Score: 0.437)
    11. European (Score: 0.437)
    12. Byzantine art (Score: 0.436)
    13. Postmodernism (Score: 0.435)
    14. Eastern Europe (Score: 0.433)
    15. Cubism (Score: 0.433)
    16. Europe (Score: 0.432)
    17. Impressionism (Score: 0.432)
    18. Bauhaus (Score: 0.431)
    19. Surrealism (Score: 0.429)
    20. Expressionism (Score: 0.429)



```python
# This time we'll query using content vector
query_results = query_knn("Famous battles in Scottish history", "Articles", "content_vector")
for i, result in enumerate(query_results):
    print(f"{i + 1}. {result[2]} (Score: {round(1 - result[3], 3)})")
```

    1. Battle of Bannockburn (Score: 0.489)
    2. Wars of Scottish Independence (Score: 0.474)
    3. 1651 (Score: 0.457)
    4. First War of Scottish Independence (Score: 0.452)
    5. Robert I of Scotland (Score: 0.445)
    6. 841 (Score: 0.441)
    7. 1716 (Score: 0.441)
    8. 1314 (Score: 0.429)
    9. 1263 (Score: 0.428)
    10. William Wallace (Score: 0.426)
    11. Stirling (Score: 0.419)
    12. 1306 (Score: 0.419)
    13. 1746 (Score: 0.418)
    14. 1040s (Score: 0.414)
    15. 1106 (Score: 0.412)
    16. 1304 (Score: 0.411)
    17. David II of Scotland (Score: 0.408)
    18. Braveheart (Score: 0.407)
    19. 1124 (Score: 0.406)
    20. July 27 (Score: 0.405)



```python

```


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb


key: _node_content
string_value:
{"id_": "b935e0cf-20c0-4a23-b100-21f016a57adb", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "056e2616-a5d4-4255-96a8-bba569b14500", "node_type": "1", "metadata": {}, "hash": "da3a6cc971976443e5042af317c47553534dcf064581159176f061bf697d0812", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3150, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Using Chroma for Embeddings Search

This notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.

### What is a Vector Database

A vector database is a database made to store, manage and search embedding vectors. The use of embeddings to encode unstructured data (text, audio, video and more) as vectors for consumption by machine-learning models has exploded in recent years, due to the increasing effectiveness of AI in solving use cases involving natural language, image recognition and other unstructured forms of data. Vector databases have emerged as an effective solution for enterprises to deliver and scale these use cases.

### Why use a Vector Database

Vector databases enable enterprises to take many of the embeddings use cases we've shared in this repo (question and answering, chatbot and recommendation services, for example), and make use of them in a secure, scalable environment. Many of our customers make embeddings solve their problems at small scale but performance and security hold them back from going into production - we see vector databases as a key component in solving that, and in this guide we'll walk through the basics of embedding text data, storing it in a vector database and using it for semantic search.


### Demo Flow
The demo flow is:
- **Setup**: Import packages and set any required variables
- **Load data**: Load a dataset and embed it using OpenAI embeddings
- **Chroma**:
    - *Setup*: Here we'll set up the Python client for Chroma. For more details go [here](https://docs.trychroma.com/usage-guide)
    - *Index Data*: We'll create collections with vectors for __titles__ and __content__
    - *Search Data*: We'll run a few searches to confirm it works

Once you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings.

## Setup

Import the required libraries and set the embedding model that we'd like to use.


```python
# Make sure the OpenAI library is installed
%pip install openai

# We'll need to install the Chroma client
%pip install chromadb

# Install wget to pull zip file
%pip install wget

# Install numpy for data manipulation
%pip install numpy
```

    Collecting openai
      Obtaining dependency information for openai from https://files.pythonhosted.org/packages/67/78/7588a047e458cb8075a4089d721d7af5e143ff85a2388d4a28c530be0494/openai-0.27.8-py3-none-any.whl.metadata
      Downloading openai-0.27.8-py3-none-any.whl.metadata (13 kB)
    Collecting requests>=2.20 (from openai)
      Obtaining dependency information for requests>=2.20 from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
      Using cached requests-2.31.0-py3-none-any.whl.metadata (4.


key: _node_content
string_value:
{"id_": "056e2616-a5d4-4255-96a8-bba569b14500", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b935e0cf-20c0-4a23-b100-21f016a57adb", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "712d163f1c2c1cb788a63e3fcd3e1b2fedba0d41cdc9668c9503892a6a6ec121", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "66597835-989b-42d1-907a-969fec381a64", "node_type": "1", "metadata": {}, "hash": "494e907ac46e0d980cfdd6b423c46a7b1e30acae9fd28fc6e9a1338e69d45b32", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2287, "end_char_idx": 4380, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Make sure the OpenAI library is installed
%pip install openai

# We'll need to install the Chroma client
%pip install chromadb

# Install wget to pull zip file
%pip install wget

# Install numpy for data manipulation
%pip install numpy
```

    Collecting openai
      Obtaining dependency information for openai from https://files.pythonhosted.org/packages/67/78/7588a047e458cb8075a4089d721d7af5e143ff85a2388d4a28c530be0494/openai-0.27.8-py3-none-any.whl.metadata
      Downloading openai-0.27.8-py3-none-any.whl.metadata (13 kB)
    Collecting requests>=2.20 (from openai)
      Obtaining dependency information for requests>=2.20 from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata
      Using cached requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)
    Collecting tqdm (from openai)
      Using cached tqdm-4.65.0-py3-none-any.whl (77 kB)
    Collecting aiohttp (from openai)
      Obtaining dependency information for aiohttp from https://files.pythonhosted.org/packages/fa/9e/49002fde2a97d7df0e162e919c31cf13aa9f184537739743d1239edd0e67/aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata (7.7 kB)
    Collecting charset-normalizer<4,>=2 (from requests>=2.20->openai)
      Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/ec/a7/96835706283d63fefbbbb4f119d52f195af00fc747e67cc54397c56312c8/charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (31 kB)
    Collecting idna<4,>=2.5 (from requests>=2.20->openai)
      Using cached idna-3.4-py3-none-any.whl (61 kB)
    Collecting urllib3<3,>=1.21.1 (from requests>=2.20->openai)
      Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/9b/81/62fd61001fa4b9d0df6e31d47ff49cfa9de4af03adecf339c7bc30656b37/urllib3-2.0.4-py3-none-any.whl.


key: _node_content
string_value:
{"id_": "66597835-989b-42d1-907a-969fec381a64", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "056e2616-a5d4-4255-96a8-bba569b14500", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fe463b61c44a8eb3af0bd04df171fcbf862e4032eaaf99ad7737b516c255ce93", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8eba3e05-0831-4436-a2c1-3a7a6b057c6a", "node_type": "1", "metadata": {}, "hash": "3a85adff5038cd5128ba3a32660df4e898822687a4e911a8aa7b363160bb8c35", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3456, "end_char_idx": 5404, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl.metadata (7.7 kB)
    Collecting charset-normalizer<4,>=2 (from requests>=2.20->openai)
      Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/ec/a7/96835706283d63fefbbbb4f119d52f195af00fc747e67cc54397c56312c8/charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (31 kB)
    Collecting idna<4,>=2.5 (from requests>=2.20->openai)
      Using cached idna-3.4-py3-none-any.whl (61 kB)
    Collecting urllib3<3,>=1.21.1 (from requests>=2.20->openai)
      Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/9b/81/62fd61001fa4b9d0df6e31d47ff49cfa9de4af03adecf339c7bc30656b37/urllib3-2.0.4-py3-none-any.whl.metadata
      Downloading urllib3-2.0.4-py3-none-any.whl.metadata (6.6 kB)
    Collecting certifi>=2017.4.17 (from requests>=2.20->openai)
      Using cached certifi-2023.5.7-py3-none-any.whl (156 kB)
    Collecting attrs>=17.3.0 (from aiohttp->openai)
      Using cached attrs-23.1.0-py3-none-any.whl (61 kB)
    Collecting multidict<7.0,>=4.5 (from aiohttp->openai)
      Using cached multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl (29 kB)
    Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->openai)
      Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)
    Collecting yarl<2.0,>=1.0 (from aiohttp->openai)
      Using cached yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl (62 kB)
    Collecting frozenlist>=1.1.1 (from aiohttp->openai)
      Obtaining dependency information for frozenlist>=1.1.1 from https://files.pythonhosted.org/packages/67/6a/55a49da0fa373ac9aa49ccd5b6393ecc183e2a0904d9449ea3ee1163e0b1/frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Downloading frozenlist-1.4.


key: _node_content
string_value:
{"id_": "8eba3e05-0831-4436-a2c1-3a7a6b057c6a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "66597835-989b-42d1-907a-969fec381a64", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "5bf7250ac75635842d8521cb0812470a1faef05758e6c5f8c6ce97d961b67946", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "81dbfa72-5dc3-426c-9ddb-ed40d5fb8bba", "node_type": "1", "metadata": {}, "hash": "3a8293023104a5c65c389a14cc7d2c3875dfc149c27663b687af99e7c89d2e3e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4417, "end_char_idx": 6142, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0.4-py3-none-any.whl.metadata (6.6 kB)
    Collecting certifi>=2017.4.17 (from requests>=2.20->openai)
      Using cached certifi-2023.5.7-py3-none-any.whl (156 kB)
    Collecting attrs>=17.3.0 (from aiohttp->openai)
      Using cached attrs-23.1.0-py3-none-any.whl (61 kB)
    Collecting multidict<7.0,>=4.5 (from aiohttp->openai)
      Using cached multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl (29 kB)
    Collecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->openai)
      Using cached async_timeout-4.0.2-py3-none-any.whl (5.8 kB)
    Collecting yarl<2.0,>=1.0 (from aiohttp->openai)
      Using cached yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl (62 kB)
    Collecting frozenlist>=1.1.1 (from aiohttp->openai)
      Obtaining dependency information for frozenlist>=1.1.1 from https://files.pythonhosted.org/packages/67/6a/55a49da0fa373ac9aa49ccd5b6393ecc183e2a0904d9449ea3ee1163e0b1/frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Downloading frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.2 kB)
    Collecting aiosignal>=1.1.2 (from aiohttp->openai)
      Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
    Using cached openai-0.27.8-py3-none-any.whl (73 kB)
    Using cached requests-2.31.0-py3-none-any.whl (62 kB)
    Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl (343 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m343.9/343.9 kB[0m [31m11.4 MB/s[0m eta [36m0:00:00[0m
    [?25hUsing cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl (124 kB)
    Downloading frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl (46 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m46.0/46.0 kB[0m [31m4.


key: _node_content
string_value:
{"id_": "81dbfa72-5dc3-426c-9ddb-ed40d5fb8bba", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8eba3e05-0831-4436-a2c1-3a7a6b057c6a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3183224ab57868e1f7b74f168f8714b045de535cfbeaec8894902ecaff40cc78", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7bac77ff-1d24-48c9-bcea-32a1f482a161", "node_type": "1", "metadata": {}, "hash": "cbbb86efe862822f9bafe2078da11a3fb6af7839abee83bb576b461a70874b54", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5324, "end_char_idx": 7067, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Downloading frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.2 kB)
    Collecting aiosignal>=1.1.2 (from aiohttp->openai)
      Using cached aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
    Using cached openai-0.27.8-py3-none-any.whl (73 kB)
    Using cached requests-2.31.0-py3-none-any.whl (62 kB)
    Downloading aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl (343 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m343.9/343.9 kB[0m [31m11.4 MB/s[0m eta [36m0:00:00[0m
    [?25hUsing cached charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl (124 kB)
    Downloading frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl (46 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m46.0/46.0 kB[0m [31m4.4 MB/s[0m eta [36m0:00:00[0m
    [?25hDownloading urllib3-2.0.4-py3-none-any.whl (123 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m123.9/123.9 kB[0m [31m20.0 MB/s[0m eta [36m0:00:00[0m
    [?25hInstalling collected packages: urllib3, tqdm, multidict, idna, frozenlist, charset-normalizer, certifi, attrs, async-timeout, yarl, requests, aiosignal, aiohttp, openai
    Successfully installed aiohttp-3.8.5 aiosignal-1.3.1 async-timeout-4.0.2 attrs-23.1.0 certifi-2023.5.7 charset-normalizer-3.2.0 frozenlist-1.4.0 idna-3.4 multidict-6.0.4 openai-0.27.8 requests-2.31.0 tqdm-4.65.0 urllib3-2.0.4 yarl-1.9.2
    Note: you may need to restart the kernel to use updated packages.
    Collecting chromadb
      Obtaining dependency information for chromadb from https://files.pythonhosted.org/packages/47/b7/41d975f02818c965cdb8a119cab5a38cfb08e0c1abb18efebe9a373ea97b/chromadb-0.4.2-py3-none-any.


key: _node_content
string_value:
{"id_": "7bac77ff-1d24-48c9-bcea-32a1f482a161", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "81dbfa72-5dc3-426c-9ddb-ed40d5fb8bba", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "1bb8f0b915b789dc47ed17f5aca6414674ff23d44854e36f8d8c4b1c92678faf", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8e58a4e6-06c7-4d64-9d9d-702823f0bdab", "node_type": "1", "metadata": {}, "hash": "f0bff1d46574a874bc313737bc10eea73393130c9a01b4ed313b415ebcd929ea", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6121, "end_char_idx": 8015, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0/46.0 kB[0m [31m4.4 MB/s[0m eta [36m0:00:00[0m
    [?25hDownloading urllib3-2.0.4-py3-none-any.whl (123 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m123.9/123.9 kB[0m [31m20.0 MB/s[0m eta [36m0:00:00[0m
    [?25hInstalling collected packages: urllib3, tqdm, multidict, idna, frozenlist, charset-normalizer, certifi, attrs, async-timeout, yarl, requests, aiosignal, aiohttp, openai
    Successfully installed aiohttp-3.8.5 aiosignal-1.3.1 async-timeout-4.0.2 attrs-23.1.0 certifi-2023.5.7 charset-normalizer-3.2.0 frozenlist-1.4.0 idna-3.4 multidict-6.0.4 openai-0.27.8 requests-2.31.0 tqdm-4.65.0 urllib3-2.0.4 yarl-1.9.2
    Note: you may need to restart the kernel to use updated packages.
    Collecting chromadb
      Obtaining dependency information for chromadb from https://files.pythonhosted.org/packages/47/b7/41d975f02818c965cdb8a119cab5a38cfb08e0c1abb18efebe9a373ea97b/chromadb-0.4.2-py3-none-any.whl.metadata
      Downloading chromadb-0.4.2-py3-none-any.whl.metadata (6.9 kB)
    Collecting pandas>=1.3 (from chromadb)
      Obtaining dependency information for pandas>=1.3 from https://files.pythonhosted.org/packages/4a/f6/f620ca62365d83e663a255a41b08d2fc2eaf304e0b8b21bb6d62a7390fe3/pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl.metadata (18 kB)
    Requirement already satisfied: requests>=2.28 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (2.31.0)
    Collecting pydantic<2.0,>=1.9 (from chromadb)
      Obtaining dependency information for pydantic<2.0,>=1.9 from https://files.pythonhosted.org/packages/79/3e/6b4d0fb2174beceac9a991ba8e67158b45c35faca9ea4545ae32d47096cd/pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.


key: _node_content
string_value:
{"id_": "8e58a4e6-06c7-4d64-9d9d-702823f0bdab", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7bac77ff-1d24-48c9-bcea-32a1f482a161", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "57367f8bcb998b4bdec07caa2bb404c03a5fe0a707eee56eb12240cfa59a8a4b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f17390e6-ccb0-43b3-aabd-5bd5f800515c", "node_type": "1", "metadata": {}, "hash": "a747c43aaf3aa919f95211780385a6dba693592494771bcde4005c8b729b5b67", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7050, "end_char_idx": 8929, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
4.2-py3-none-any.whl.metadata
      Downloading chromadb-0.4.2-py3-none-any.whl.metadata (6.9 kB)
    Collecting pandas>=1.3 (from chromadb)
      Obtaining dependency information for pandas>=1.3 from https://files.pythonhosted.org/packages/4a/f6/f620ca62365d83e663a255a41b08d2fc2eaf304e0b8b21bb6d62a7390fe3/pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl.metadata (18 kB)
    Requirement already satisfied: requests>=2.28 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (2.31.0)
    Collecting pydantic<2.0,>=1.9 (from chromadb)
      Obtaining dependency information for pydantic<2.0,>=1.9 from https://files.pythonhosted.org/packages/79/3e/6b4d0fb2174beceac9a991ba8e67158b45c35faca9ea4545ae32d47096cd/pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl.metadata (148 kB)
    Collecting chroma-hnswlib==0.7.1 (from chromadb)
      Obtaining dependency information for chroma-hnswlib==0.7.1 from https://files.pythonhosted.org/packages/a5/d5/54947127f5cb2a1fcef40877fb3e6044495eec0a158ba0956babe4ab2a77/chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl.metadata
      Using cached chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl.metadata (252 bytes)
    Collecting fastapi<0.100.0,>=0.95.2 (from chromadb)
      Obtaining dependency information for fastapi<0.100.0,>=0.95.2 from https://files.pythonhosted.org/packages/73/eb/03b691afa0b5ffa1e93ed34f97ec1e7855c758efbdcfb16c209af0b0506b/fastapi-0.99.1-py3-none-any.whl.metadata
      Using cached fastapi-0.99.1-py3-none-any.whl.metadata (23 kB)
    Collecting uvicorn[standard]>=0.18.3 (from chromadb)
      Obtaining dependency information for uvicorn[standard]>=0.18.3 from https://files.pythonhosted.


key: _node_content
string_value:
{"id_": "f17390e6-ccb0-43b3-aabd-5bd5f800515c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8e58a4e6-06c7-4d64-9d9d-702823f0bdab", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "21722b67ffa496f9d8e8a8876a692e0cc74e02df264be26c0797895ab765d41e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9a73e628-5103-458d-a5e6-bd9f586aeae4", "node_type": "1", "metadata": {}, "hash": "63a74415aa0316047bba32682a9b6ff1252e608c967c6a44c612e413f620213f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7903, "end_char_idx": 9906, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
11-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl.metadata (148 kB)
    Collecting chroma-hnswlib==0.7.1 (from chromadb)
      Obtaining dependency information for chroma-hnswlib==0.7.1 from https://files.pythonhosted.org/packages/a5/d5/54947127f5cb2a1fcef40877fb3e6044495eec0a158ba0956babe4ab2a77/chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl.metadata
      Using cached chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl.metadata (252 bytes)
    Collecting fastapi<0.100.0,>=0.95.2 (from chromadb)
      Obtaining dependency information for fastapi<0.100.0,>=0.95.2 from https://files.pythonhosted.org/packages/73/eb/03b691afa0b5ffa1e93ed34f97ec1e7855c758efbdcfb16c209af0b0506b/fastapi-0.99.1-py3-none-any.whl.metadata
      Using cached fastapi-0.99.1-py3-none-any.whl.metadata (23 kB)
    Collecting uvicorn[standard]>=0.18.3 (from chromadb)
      Obtaining dependency information for uvicorn[standard]>=0.18.3 from https://files.pythonhosted.org/packages/5d/07/b9eac057f7efa56900640a233c1ed63db83568322c6bcbabe98f741d5289/uvicorn-0.23.1-py3-none-any.whl.metadata
      Using cached uvicorn-0.23.1-py3-none-any.whl.metadata (6.2 kB)
    Collecting numpy>=1.21.6 (from chromadb)
      Obtaining dependency information for numpy>=1.21.6 from https://files.pythonhosted.org/packages/1b/cd/9e8313ffd849626c836fffd7881296a74f53a7739bd9ce7a6e22b1fc843b/numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.6 kB)
    Collecting posthog>=2.4.0 (from chromadb)
      Using cached posthog-3.0.1-py2.py3-none-any.whl (37 kB)
    Requirement already satisfied: typing-extensions>=4.5.0 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (4.7.1)
    Collecting pulsar-client>=3.1.0 (from chromadb)
      Obtaining dependency information for pulsar-client>=3.1.0 from https://files.pythonhosted.


key: _node_content
string_value:
{"id_": "9a73e628-5103-458d-a5e6-bd9f586aeae4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f17390e6-ccb0-43b3-aabd-5bd5f800515c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "828bf73fee962d40b49987729ec15243a4626889a2e15e526ca281f0ae72c8f1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1c2d3419-b938-485c-83ef-57bf79ac529e", "node_type": "1", "metadata": {}, "hash": "31e5bc06ffe5c69f796c05e65e3e7d768b09ec188e5420d2f9a2d54a84f3f413", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8810, "end_char_idx": 10806, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
3 (from chromadb)
      Obtaining dependency information for uvicorn[standard]>=0.18.3 from https://files.pythonhosted.org/packages/5d/07/b9eac057f7efa56900640a233c1ed63db83568322c6bcbabe98f741d5289/uvicorn-0.23.1-py3-none-any.whl.metadata
      Using cached uvicorn-0.23.1-py3-none-any.whl.metadata (6.2 kB)
    Collecting numpy>=1.21.6 (from chromadb)
      Obtaining dependency information for numpy>=1.21.6 from https://files.pythonhosted.org/packages/1b/cd/9e8313ffd849626c836fffd7881296a74f53a7739bd9ce7a6e22b1fc843b/numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (5.6 kB)
    Collecting posthog>=2.4.0 (from chromadb)
      Using cached posthog-3.0.1-py2.py3-none-any.whl (37 kB)
    Requirement already satisfied: typing-extensions>=4.5.0 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (4.7.1)
    Collecting pulsar-client>=3.1.0 (from chromadb)
      Obtaining dependency information for pulsar-client>=3.1.0 from https://files.pythonhosted.org/packages/43/85/ab0455008ce3335a1c75a7c500fd8921ab166f34821fa67dc91ae9687a40/pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl.metadata
      Using cached pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl.metadata (1.0 kB)
    Collecting onnxruntime>=1.14.1 (from chromadb)
      Obtaining dependency information for onnxruntime>=1.14.1 from https://files.pythonhosted.org/packages/cf/06/0c6e355b9ddbebc34d0e21bc5be1e4bd2c124ebd9030525838fa6e65eaa8/onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (4.0 kB)
    Collecting tokenizers>=0.13.2 (from chromadb)
      Using cached tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl (3.9 MB)
    Collecting pypika>=0.48.9 (from chromadb)
      Using cached PyPika-0.48.9-py2.py3-none-any.whl
    Requirement already satisfied: tqdm>=4.65.


key: _node_content
string_value:
{"id_": "1c2d3419-b938-485c-83ef-57bf79ac529e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9a73e628-5103-458d-a5e6-bd9f586aeae4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "72e29c6be555eab86497a73b53edaf55a16a54e1705bf4df788ea1a8098c5cde", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c20f8957-3544-4404-9927-15ddfa054af4", "node_type": "1", "metadata": {}, "hash": "a35813e3cee8b0f51f9ff668bfae389c2851389d3717cdb4e0bcd321fabf5503", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9790, "end_char_idx": 11821, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
1.0 (from chromadb)
      Obtaining dependency information for pulsar-client>=3.1.0 from https://files.pythonhosted.org/packages/43/85/ab0455008ce3335a1c75a7c500fd8921ab166f34821fa67dc91ae9687a40/pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl.metadata
      Using cached pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl.metadata (1.0 kB)
    Collecting onnxruntime>=1.14.1 (from chromadb)
      Obtaining dependency information for onnxruntime>=1.14.1 from https://files.pythonhosted.org/packages/cf/06/0c6e355b9ddbebc34d0e21bc5be1e4bd2c124ebd9030525838fa6e65eaa8/onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (4.0 kB)
    Collecting tokenizers>=0.13.2 (from chromadb)
      Using cached tokenizers-0.13.3-cp310-cp310-macosx_12_0_arm64.whl (3.9 MB)
    Collecting pypika>=0.48.9 (from chromadb)
      Using cached PyPika-0.48.9-py2.py3-none-any.whl
    Requirement already satisfied: tqdm>=4.65.0 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (4.65.0)
    Collecting overrides>=7.3.1 (from chromadb)
      Using cached overrides-7.3.1-py3-none-any.whl (17 kB)
    Collecting importlib-resources (from chromadb)
      Obtaining dependency information for importlib-resources from https://files.pythonhosted.org/packages/29/d1/bed03eca30aa05aaf6e0873de091f9385c48705c4a607c2dfe3edbe543e8/importlib_resources-6.0.0-py3-none-any.whl.metadata
      Using cached importlib_resources-6.0.0-py3-none-any.whl.metadata (4.2 kB)
    Collecting starlette<0.28.0,>=0.27.0 (from fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for starlette<0.28.0,>=0.27.0 from https://files.pythonhosted.org/packages/58/f8/e2cca22387965584a409795913b774235752be4176d276714e15e1a58884/starlette-0.27.0-py3-none-any.whl.metadata
      Using cached starlette-0.27.0-py3-none-any.whl.metadata (5.8 kB)
    Collecting coloredlogs (from onnxruntime>=1.14.


key: _node_content
string_value:
{"id_": "c20f8957-3544-4404-9927-15ddfa054af4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1c2d3419-b938-485c-83ef-57bf79ac529e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "031b9c8f34ad1821435ae6571ee8b816be4d5bfdcf82d45c4e8cd5566cc300e6", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e8d9ad6a-14da-46f7-ab3b-d7a3e766879d", "node_type": "1", "metadata": {}, "hash": "a9771477b207bbed33c5eec2fcbe187b1b39d6c7b39c7b4079f475af7dbab9cc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10734, "end_char_idx": 12870, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
48.9-py2.py3-none-any.whl
    Requirement already satisfied: tqdm>=4.65.0 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from chromadb) (4.65.0)
    Collecting overrides>=7.3.1 (from chromadb)
      Using cached overrides-7.3.1-py3-none-any.whl (17 kB)
    Collecting importlib-resources (from chromadb)
      Obtaining dependency information for importlib-resources from https://files.pythonhosted.org/packages/29/d1/bed03eca30aa05aaf6e0873de091f9385c48705c4a607c2dfe3edbe543e8/importlib_resources-6.0.0-py3-none-any.whl.metadata
      Using cached importlib_resources-6.0.0-py3-none-any.whl.metadata (4.2 kB)
    Collecting starlette<0.28.0,>=0.27.0 (from fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for starlette<0.28.0,>=0.27.0 from https://files.pythonhosted.org/packages/58/f8/e2cca22387965584a409795913b774235752be4176d276714e15e1a58884/starlette-0.27.0-py3-none-any.whl.metadata
      Using cached starlette-0.27.0-py3-none-any.whl.metadata (5.8 kB)
    Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)
      Using cached coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)
    Collecting flatbuffers (from onnxruntime>=1.14.1->chromadb)
      Obtaining dependency information for flatbuffers from https://files.pythonhosted.org/packages/6f/12/d5c79ee252793ffe845d58a913197bfa02ae9a0b5c9bc3dc4b58d477b9e7/flatbuffers-23.5.26-py2.py3-none-any.whl.metadata
      Using cached flatbuffers-23.5.26-py2.py3-none-any.whl.metadata (850 bytes)
    Requirement already satisfied: packaging in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (23.1)
    Collecting protobuf (from onnxruntime>=1.14.1->chromadb)
      Obtaining dependency information for protobuf from https://files.pythonhosted.org/packages/cb/d3/a164038605494d49acc4f9cda1c0bc200b96382c53edd561387263bb181d/protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl.metadata
      Using cached protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl.metadata (540 bytes)
    Collecting sympy (from onnxruntime>=1.14.


key: _node_content
string_value:
{"id_": "e8d9ad6a-14da-46f7-ab3b-d7a3e766879d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c20f8957-3544-4404-9927-15ddfa054af4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b5b542927892ef31a0faebb5247e4e457d78e42331639f76fcdbc0bf66228ed9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "20631516-73b8-45e5-aed3-1cd3b88a7997", "node_type": "1", "metadata": {}, "hash": "0c870e984ba4ac3159bd8b49355cba154ae47ec5b1d2e520684c6cae098b0c69", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11748, "end_char_idx": 13910, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
whl.metadata (5.8 kB)
    Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)
      Using cached coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)
    Collecting flatbuffers (from onnxruntime>=1.14.1->chromadb)
      Obtaining dependency information for flatbuffers from https://files.pythonhosted.org/packages/6f/12/d5c79ee252793ffe845d58a913197bfa02ae9a0b5c9bc3dc4b58d477b9e7/flatbuffers-23.5.26-py2.py3-none-any.whl.metadata
      Using cached flatbuffers-23.5.26-py2.py3-none-any.whl.metadata (850 bytes)
    Requirement already satisfied: packaging in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (23.1)
    Collecting protobuf (from onnxruntime>=1.14.1->chromadb)
      Obtaining dependency information for protobuf from https://files.pythonhosted.org/packages/cb/d3/a164038605494d49acc4f9cda1c0bc200b96382c53edd561387263bb181d/protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl.metadata
      Using cached protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl.metadata (540 bytes)
    Collecting sympy (from onnxruntime>=1.14.1->chromadb)
      Using cached sympy-1.12-py3-none-any.whl (5.7 MB)
    Requirement already satisfied: python-dateutil>=2.8.2 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from pandas>=1.3->chromadb) (2.8.2)
    Collecting pytz>=2020.1 (from pandas>=1.3->chromadb)
      Using cached pytz-2023.3-py2.py3-none-any.whl (502 kB)
    Collecting tzdata>=2022.1 (from pandas>=1.3->chromadb)
      Using cached tzdata-2023.3-py2.py3-none-any.whl (341 kB)
    Requirement already satisfied: six>=1.5 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.16.0)
    Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)
      Using cached monotonic-1.6-py2.py3-none-any.whl (8.2 kB)
    Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)
      Using cached backoff-2.2.1-py3-none-any.whl (15 kB)
    Requirement already satisfied: certifi in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.


key: _node_content
string_value:
{"id_": "20631516-73b8-45e5-aed3-1cd3b88a7997", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e8d9ad6a-14da-46f7-ab3b-d7a3e766879d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "e96b6263a2c5130a5c9b3a102b6d0dd189f73875534b4b88b2c4cc4ab7898ab7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9a0eaebb-45cc-47fa-b617-594f520ba7fe", "node_type": "1", "metadata": {}, "hash": "a89dd86bd6173f8cf6ca5d19f2ea4f78e3482602aae1162d619b5ac1b4ea8642", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12800, "end_char_idx": 14928, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
whl.metadata (540 bytes)
    Collecting sympy (from onnxruntime>=1.14.1->chromadb)
      Using cached sympy-1.12-py3-none-any.whl (5.7 MB)
    Requirement already satisfied: python-dateutil>=2.8.2 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from pandas>=1.3->chromadb) (2.8.2)
    Collecting pytz>=2020.1 (from pandas>=1.3->chromadb)
      Using cached pytz-2023.3-py2.py3-none-any.whl (502 kB)
    Collecting tzdata>=2022.1 (from pandas>=1.3->chromadb)
      Using cached tzdata-2023.3-py2.py3-none-any.whl (341 kB)
    Requirement already satisfied: six>=1.5 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.16.0)
    Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)
      Using cached monotonic-1.6-py2.py3-none-any.whl (8.2 kB)
    Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)
      Using cached backoff-2.2.1-py3-none-any.whl (15 kB)
    Requirement already satisfied: certifi in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from pulsar-client>=3.1.0->chromadb) (2023.5.7)
    Requirement already satisfied: charset-normalizer<4,>=2 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.2.0)
    Requirement already satisfied: idna<4,>=2.5 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.4)
    Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (2.0.4)
    Collecting click>=7.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for click>=7.0 from https://files.pythonhosted.org/packages/1a/70/e63223f8116931d365993d4a6b7ef653a4d920b41d03de7c59499962821f/click-8.1.6-py3-none-any.whl.metadata
      Using cached click-8.1.6-py3-none-any.whl.metadata (3.0 kB)
    Collecting h11>=0.8 (from uvicorn[standard]>=0.18.


key: _node_content
string_value:
{"id_": "9a0eaebb-45cc-47fa-b617-594f520ba7fe", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "20631516-73b8-45e5-aed3-1cd3b88a7997", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "67e92f787292ab99157f792b7893d834bbe64a8ec5fb6d0b9fd7e11b3a62de28", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5ba613c4-7b47-426d-9d4c-9137306049ad", "node_type": "1", "metadata": {}, "hash": "4d4960667d4c09bc9bad056f6c3e5c657eb02277118bfed7992df10cfc65cdec", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13910, "end_char_idx": 15912, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
10/site-packages (from pulsar-client>=3.1.0->chromadb) (2023.5.7)
    Requirement already satisfied: charset-normalizer<4,>=2 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.2.0)
    Requirement already satisfied: idna<4,>=2.5 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.4)
    Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (from requests>=2.28->chromadb) (2.0.4)
    Collecting click>=7.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for click>=7.0 from https://files.pythonhosted.org/packages/1a/70/e63223f8116931d365993d4a6b7ef653a4d920b41d03de7c59499962821f/click-8.1.6-py3-none-any.whl.metadata
      Using cached click-8.1.6-py3-none-any.whl.metadata (3.0 kB)
    Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached h11-0.14.0-py3-none-any.whl (58 kB)
    Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for httptools>=0.5.0 from https://files.pythonhosted.org/packages/8f/71/d535e9f6967958d21b8fe1baeb7efb6304b86e8fcff44d0bda8690e0aec9/httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl.metadata
      Using cached httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl.metadata (3.6 kB)
    Collecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached python_dotenv-1.0.0-py3-none-any.whl (19 kB)
    Collecting pyyaml>=5.1 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for pyyaml>=5.1 from https://files.pythonhosted.org/packages/5b/07/10033a403b23405a8fc48975444463d3d10a5c2736b7eb2550b07b367429/PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (2.


key: _node_content
string_value:
{"id_": "5ba613c4-7b47-426d-9d4c-9137306049ad", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9a0eaebb-45cc-47fa-b617-594f520ba7fe", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8c5a59adb1728a915245929e31a77a048e968586ac74d3fc8abca2cf58e2e21a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "197a2123-a39c-4f9b-8eda-191c1bf8ccd8", "node_type": "1", "metadata": {}, "hash": "d3d57c5eba9d2e0f23e4154c915b76b48b8d72298d685642b4ca716a0f19d8d4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14928, "end_char_idx": 16767, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
3->chromadb)
      Using cached h11-0.14.0-py3-none-any.whl (58 kB)
    Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for httptools>=0.5.0 from https://files.pythonhosted.org/packages/8f/71/d535e9f6967958d21b8fe1baeb7efb6304b86e8fcff44d0bda8690e0aec9/httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl.metadata
      Using cached httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl.metadata (3.6 kB)
    Collecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached python_dotenv-1.0.0-py3-none-any.whl (19 kB)
    Collecting pyyaml>=5.1 (from uvicorn[standard]>=0.18.3->chromadb)
      Obtaining dependency information for pyyaml>=5.1 from https://files.pythonhosted.org/packages/5b/07/10033a403b23405a8fc48975444463d3d10a5c2736b7eb2550b07b367429/PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl.metadata
      Using cached PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (2.1 kB)
    Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl (2.1 MB)
    Collecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached watchfiles-0.19.0-cp37-abi3-macosx_11_0_arm64.whl (388 kB)
    Collecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached websockets-11.0.3-cp310-cp310-macosx_11_0_arm64.whl (121 kB)
    Collecting anyio<5,>=3.4.0 (from starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for anyio<5,>=3.4.0 from https://files.pythonhosted.org/packages/19/24/44299477fe7dcc9cb58d0a57d5a7588d6af2ff403fdd2d47a246c91a3246/anyio-3.7.1-py3-none-any.whl.metadata
      Using cached anyio-3.7.1-py3-none-any.whl.metadata (4.


key: _node_content
string_value:
{"id_": "197a2123-a39c-4f9b-8eda-191c1bf8ccd8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5ba613c4-7b47-426d-9d4c-9137306049ad", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "2a61254633a16831d1fd46a8cdd7d21bc1749a661e71d44998c53b35463ae29c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0bbe83ae-a6b1-4a6c-a86f-2de559cd8195", "node_type": "1", "metadata": {}, "hash": "532a6220d13418b7ed177e24bc801f08a6a339411958723f50510aeee323d0ef", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15821, "end_char_idx": 17678, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
whl.metadata
      Using cached PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (2.1 kB)
    Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl (2.1 MB)
    Collecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached watchfiles-0.19.0-cp37-abi3-macosx_11_0_arm64.whl (388 kB)
    Collecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)
      Using cached websockets-11.0.3-cp310-cp310-macosx_11_0_arm64.whl (121 kB)
    Collecting anyio<5,>=3.4.0 (from starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for anyio<5,>=3.4.0 from https://files.pythonhosted.org/packages/19/24/44299477fe7dcc9cb58d0a57d5a7588d6af2ff403fdd2d47a246c91a3246/anyio-3.7.1-py3-none-any.whl.metadata
      Using cached anyio-3.7.1-py3-none-any.whl.metadata (4.7 kB)
    Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)
      Using cached humanfriendly-10.0-py2.py3-none-any.whl (86 kB)
    Collecting mpmath>=0.19 (from sympy->onnxruntime>=1.14.1->chromadb)
      Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)
    Collecting sniffio>=1.1 (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Using cached sniffio-1.3.0-py3-none-any.whl (10 kB)
    Collecting exceptiongroup (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for exceptiongroup from https://files.pythonhosted.org/packages/fe/17/f43b7c9ccf399d72038042ee72785c305f6c6fdc6231942f8ab99d995742/exceptiongroup-1.1.2-py3-none-any.whl.metadata
      Using cached exceptiongroup-1.1.2-py3-none-any.whl.metadata (6.1 kB)
    Downloading chromadb-0.4.2-py3-none-any.


key: _node_content
string_value:
{"id_": "0bbe83ae-a6b1-4a6c-a86f-2de559cd8195", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "197a2123-a39c-4f9b-8eda-191c1bf8ccd8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "43bf7045ae076615ddd241a374727713a0d71da849a9d6e669dad81a3da3432e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e26ab46e-ebf3-4b7d-87a5-0f8fa8a8e492", "node_type": "1", "metadata": {}, "hash": "fc48370691c2af01a41535cb3dc824f9ffb04d104cbfc3a7c4260dc312e6f4ca", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16677, "end_char_idx": 18461, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
7.1-py3-none-any.whl.metadata
      Using cached anyio-3.7.1-py3-none-any.whl.metadata (4.7 kB)
    Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)
      Using cached humanfriendly-10.0-py2.py3-none-any.whl (86 kB)
    Collecting mpmath>=0.19 (from sympy->onnxruntime>=1.14.1->chromadb)
      Using cached mpmath-1.3.0-py3-none-any.whl (536 kB)
    Collecting sniffio>=1.1 (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Using cached sniffio-1.3.0-py3-none-any.whl (10 kB)
    Collecting exceptiongroup (from anyio<5,>=3.4.0->starlette<0.28.0,>=0.27.0->fastapi<0.100.0,>=0.95.2->chromadb)
      Obtaining dependency information for exceptiongroup from https://files.pythonhosted.org/packages/fe/17/f43b7c9ccf399d72038042ee72785c305f6c6fdc6231942f8ab99d995742/exceptiongroup-1.1.2-py3-none-any.whl.metadata
      Using cached exceptiongroup-1.1.2-py3-none-any.whl.metadata (6.1 kB)
    Downloading chromadb-0.4.2-py3-none-any.whl (399 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m399.3/399.3 kB[0m [31m12.8 MB/s[0m eta [36m0:00:00[0m
    [?25hUsing cached chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl (195 kB)
    Using cached fastapi-0.99.1-py3-none-any.whl (58 kB)
    Using cached numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl (14.0 MB)
    Using cached onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl (6.1 MB)
    Using cached pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl (10.8 MB)
    Using cached pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl (10.8 MB)
    Using cached pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl (2.5 MB)
    Using cached importlib_resources-6.0.0-py3-none-any.whl (31 kB)
    Using cached click-8.1.6-py3-none-any.


key: _node_content
string_value:
{"id_": "e26ab46e-ebf3-4b7d-87a5-0f8fa8a8e492", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0bbe83ae-a6b1-4a6c-a86f-2de559cd8195", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6c631432a4fd44f8849d95a92bac843ee984095a43aa88bf26a65aee7b5a2c9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b6f35b47-e3f9-4759-ba22-15794970a214", "node_type": "1", "metadata": {}, "hash": "be62f03cdee8aa6b42d698d98c708d7dcfddc1eb2c2ce61ffa020c5c87f7cea7", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 17595, "end_char_idx": 19469, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
1.2-py3-none-any.whl.metadata (6.1 kB)
    Downloading chromadb-0.4.2-py3-none-any.whl (399 kB)
    [2K   [90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━[0m [32m399.3/399.3 kB[0m [31m12.8 MB/s[0m eta [36m0:00:00[0m
    [?25hUsing cached chroma_hnswlib-0.7.1-cp310-cp310-macosx_13_0_arm64.whl (195 kB)
    Using cached fastapi-0.99.1-py3-none-any.whl (58 kB)
    Using cached numpy-1.25.1-cp310-cp310-macosx_11_0_arm64.whl (14.0 MB)
    Using cached onnxruntime-1.15.1-cp310-cp310-macosx_11_0_arm64.whl (6.1 MB)
    Using cached pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl (10.8 MB)
    Using cached pulsar_client-3.2.0-cp310-cp310-macosx_10_15_universal2.whl (10.8 MB)
    Using cached pydantic-1.10.11-cp310-cp310-macosx_11_0_arm64.whl (2.5 MB)
    Using cached importlib_resources-6.0.0-py3-none-any.whl (31 kB)
    Using cached click-8.1.6-py3-none-any.whl (97 kB)
    Using cached httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl (237 kB)
    Using cached PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl (169 kB)
    Using cached starlette-0.27.0-py3-none-any.whl (66 kB)
    Using cached flatbuffers-23.5.26-py2.py3-none-any.whl (26 kB)
    Using cached protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl (400 kB)
    Using cached uvicorn-0.23.1-py3-none-any.whl (59 kB)
    Using cached anyio-3.7.1-py3-none-any.whl (80 kB)
    Using cached exceptiongroup-1.1.2-py3-none-any.whl (14 kB)
    Installing collected packages: tokenizers, pytz, pypika, mpmath, monotonic, flatbuffers, websockets, uvloop, tzdata, sympy, sniffio, pyyaml, python-dotenv, pydantic, pulsar-client, protobuf, overrides, numpy, importlib-resources, humanfriendly, httptools, h11, exceptiongroup, click, backoff, uvicorn, posthog, pandas, coloredlogs, chroma-hnswlib, anyio, watchfiles, starlette, onnxruntime, fastapi, chromadb
    Successfully installed anyio-3.7.1 backoff-2.2.


key: _node_content
string_value:
{"id_": "b6f35b47-e3f9-4759-ba22-15794970a214", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e26ab46e-ebf3-4b7d-87a5-0f8fa8a8e492", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "eefa500abf2c84b6c61c909f2583dd50554d96b4d7ab0f955acf8f8100976f97", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b7c035a2-45e1-4111-a572-58aa56b32d3a", "node_type": "1", "metadata": {}, "hash": "0648dffc3582738fbfdcfe4d638c81845d49076868d83d09ff3cb1ab80c6b817", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 18390, "end_char_idx": 20088, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0.0-py3-none-any.whl (31 kB)
    Using cached click-8.1.6-py3-none-any.whl (97 kB)
    Using cached httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl (237 kB)
    Using cached PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl (169 kB)
    Using cached starlette-0.27.0-py3-none-any.whl (66 kB)
    Using cached flatbuffers-23.5.26-py2.py3-none-any.whl (26 kB)
    Using cached protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl (400 kB)
    Using cached uvicorn-0.23.1-py3-none-any.whl (59 kB)
    Using cached anyio-3.7.1-py3-none-any.whl (80 kB)
    Using cached exceptiongroup-1.1.2-py3-none-any.whl (14 kB)
    Installing collected packages: tokenizers, pytz, pypika, mpmath, monotonic, flatbuffers, websockets, uvloop, tzdata, sympy, sniffio, pyyaml, python-dotenv, pydantic, pulsar-client, protobuf, overrides, numpy, importlib-resources, humanfriendly, httptools, h11, exceptiongroup, click, backoff, uvicorn, posthog, pandas, coloredlogs, chroma-hnswlib, anyio, watchfiles, starlette, onnxruntime, fastapi, chromadb
    Successfully installed anyio-3.7.1 backoff-2.2.1 chroma-hnswlib-0.7.1 chromadb-0.4.2 click-8.1.6 coloredlogs-15.0.1 exceptiongroup-1.1.2 fastapi-0.99.1 flatbuffers-23.5.26 h11-0.14.0 httptools-0.6.0 humanfriendly-10.0 importlib-resources-6.0.0 monotonic-1.6 mpmath-1.3.0 numpy-1.25.1 onnxruntime-1.15.1 overrides-7.3.1 pandas-2.0.3 posthog-3.0.1 protobuf-4.23.4 pulsar-client-3.2.0 pydantic-1.10.11 pypika-0.48.9 python-dotenv-1.0.0 pytz-2023.3 pyyaml-6.0.1 sniffio-1.3.0 starlette-0.27.0 sympy-1.12 tokenizers-0.13.3 tzdata-2023.3 uvicorn-0.23.1 uvloop-0.17.0 watchfiles-0.19.0 websockets-11.0.3
    Note: you may need to restart the kernel to use updated packages.


key: _node_content
string_value:
{"id_": "b7c035a2-45e1-4111-a572-58aa56b32d3a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b6f35b47-e3f9-4759-ba22-15794970a214", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "6e16a96eea041d876d6b2c6736e6c7515f0109628d44896f40ef1ee99470ba2d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "78fef95d-4d0d-48b3-84c3-613a627b65b5", "node_type": "1", "metadata": {}, "hash": "bb97a35ff3877fb498af5413fb94a64d443cfb7e6fa92291c630aa3306f2ccca", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 19110, "end_char_idx": 20985, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
uvloop, tzdata, sympy, sniffio, pyyaml, python-dotenv, pydantic, pulsar-client, protobuf, overrides, numpy, importlib-resources, humanfriendly, httptools, h11, exceptiongroup, click, backoff, uvicorn, posthog, pandas, coloredlogs, chroma-hnswlib, anyio, watchfiles, starlette, onnxruntime, fastapi, chromadb
    Successfully installed anyio-3.7.1 backoff-2.2.1 chroma-hnswlib-0.7.1 chromadb-0.4.2 click-8.1.6 coloredlogs-15.0.1 exceptiongroup-1.1.2 fastapi-0.99.1 flatbuffers-23.5.26 h11-0.14.0 httptools-0.6.0 humanfriendly-10.0 importlib-resources-6.0.0 monotonic-1.6 mpmath-1.3.0 numpy-1.25.1 onnxruntime-1.15.1 overrides-7.3.1 pandas-2.0.3 posthog-3.0.1 protobuf-4.23.4 pulsar-client-3.2.0 pydantic-1.10.11 pypika-0.48.9 python-dotenv-1.0.0 pytz-2023.3 pyyaml-6.0.1 sniffio-1.3.0 starlette-0.27.0 sympy-1.12 tokenizers-0.13.3 tzdata-2023.3 uvicorn-0.23.1 uvloop-0.17.0 watchfiles-0.19.0 websockets-11.0.3
    Note: you may need to restart the kernel to use updated packages.
    Collecting wget
      Using cached wget-3.2.zip (10 kB)
      Preparing metadata (setup.py) ... [?25ldone
    [?25hBuilding wheels for collected packages: wget
      Building wheel for wget (setup.py) ... [?25ldone
    [?25h  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9657 sha256=b2d83c5fcdeab398d0a4e9808a470bbf725fffea4a6130e731c6097b9561005b
      Stored in directory: /Users/antontroynikov/Library/Caches/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769
    Successfully built wget
    Installing collected packages: wget
    Successfully installed wget-3.2
    Note: you may need to restart the kernel to use updated packages.
    Requirement already satisfied: numpy in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (1.25.1)
    Note: you may need to restart the kernel to use updated packages.


key: _node_content
string_value:
{"id_": "78fef95d-4d0d-48b3-84c3-613a627b65b5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b7c035a2-45e1-4111-a572-58aa56b32d3a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a694c829048ae6db02e60c5b4e5721efeef0ec63277083f57de3fb15e770e9f3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f4164929-d6ce-44b5-9cc4-b0e6005c6b2e", "node_type": "1", "metadata": {}, "hash": "cc5552ba2aa38db470f007100d75c8ebf3eaacbb31b332e54b1079d48a6a9459", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 19851, "end_char_idx": 22189, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
0.0 pytz-2023.3 pyyaml-6.0.1 sniffio-1.3.0 starlette-0.27.0 sympy-1.12 tokenizers-0.13.3 tzdata-2023.3 uvicorn-0.23.1 uvloop-0.17.0 watchfiles-0.19.0 websockets-11.0.3
    Note: you may need to restart the kernel to use updated packages.
    Collecting wget
      Using cached wget-3.2.zip (10 kB)
      Preparing metadata (setup.py) ... [?25ldone
    [?25hBuilding wheels for collected packages: wget
      Building wheel for wget (setup.py) ... [?25ldone
    [?25h  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9657 sha256=b2d83c5fcdeab398d0a4e9808a470bbf725fffea4a6130e731c6097b9561005b
      Stored in directory: /Users/antontroynikov/Library/Caches/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769
    Successfully built wget
    Installing collected packages: wget
    Successfully installed wget-3.2
    Note: you may need to restart the kernel to use updated packages.
    Requirement already satisfied: numpy in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (1.25.1)
    Note: you may need to restart the kernel to use updated packages.



```python
import openai
import pandas as pd
import os
import wget
from ast import literal_eval

# Chroma's client library for Python
import chromadb

# I've set this to our new embeddings model, this can be changed to the embedding model of your choice
EMBEDDING_MODEL = "text-embedding-3-small"

# Ignore unclosed SSL socket warnings - optional in case you get these errors
import warnings

warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning) 
```

## Load data

In this section we'll load embedded data that we've prepared previous to this session.


```python
embeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```




    'vector_database_wikipedia_articles_embedded.zip'




```python
import zipfile
with zipfile.ZipFile("vector_database_wikipedia_articles_embedded.zip","r") as zip_ref:
    zip_ref.extractall("../data")
```


```python
article_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')
```


```python
article_df.head()
```


key: _node_content
string_value:
{"id_": "f4164929-d6ce-44b5-9cc4-b0e6005c6b2e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "78fef95d-4d0d-48b3-84c3-613a627b65b5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "bf42f226b94a5c82b0ea8c4149292011f2eff6474192952a02215daa5b86db73", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b726e3c9-eb22-4b81-a44b-f6d00daa5d4b", "node_type": "1", "metadata": {}, "hash": "55d75d734c6c465942a4bbe99c9e2550ca1dde90d5af183d906f3fc23ae149c3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 20777, "end_char_idx": 23205, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Requirement already satisfied: numpy in /Users/antontroynikov/miniforge3/envs/chroma-openai-cookbook/lib/python3.10/site-packages (1.25.1)
    Note: you may need to restart the kernel to use updated packages.



```python
import openai
import pandas as pd
import os
import wget
from ast import literal_eval

# Chroma's client library for Python
import chromadb

# I've set this to our new embeddings model, this can be changed to the embedding model of your choice
EMBEDDING_MODEL = "text-embedding-3-small"

# Ignore unclosed SSL socket warnings - optional in case you get these errors
import warnings

warnings.filterwarnings(action="ignore", message="unclosed", category=ResourceWarning)
warnings.filterwarnings("ignore", category=DeprecationWarning) 
```

## Load data

In this section we'll load embedded data that we've prepared previous to this session.


```python
embeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'

# The file is ~700 MB so this will take some time
wget.download(embeddings_url)
```




    'vector_database_wikipedia_articles_embedded.zip'




```python
import zipfile
with zipfile.ZipFile("vector_database_wikipedia_articles_embedded.zip","r") as zip_ref:
    zip_ref.extractall("../data")
```


```python
article_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')
```


```python
article_df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.)


key: _node_content
string_value:
{"id_": "b726e3c9-eb22-4b81-a44b-f6d00daa5d4b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c54ae26-1c8c-487f-bf17-dd7249ab7b6c", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "358148240794022cd40de030ff5b55b1ca1a25fbd1e78d82f880f37d127090a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f4164929-d6ce-44b5-9cc4-b0e6005c6b2e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/chroma/Using_Chroma_for_embeddings_search.ipynb", "title": "Using Chroma for Embeddings Search", "tokens": 10702, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "5ab0f70936dfab0b4238ff107eed102792d23e0750a4ec7fc444661a843d47b0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6420e2cc-8203-4f93-9d24-e0701e68c45b", "node_type": "1", "metadata": {}, "hash": "0eb9f1fbcad7c87e0ae1cc80d55f22c888212d66f0901dbcd88725f848899c3c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 22058, "end_char_idx": 24236, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
article_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')
```


```python
article_df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>id</th>
      <th>url</th>
      <th>title</th>
      <th>text</th>
      <th>title_vector</th>
      <th>content_vector</th>
      <th>vector_id</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>1</td>
      <td>https://simple.wikipedia.org/wiki/April</td>
      <td>April</td>
      <td>April is the fourth month of the year in the J...</td>
      <td>[0.001009464613161981, -0.020700545981526375, ...</td>
      <td>[-0.011253940872848034, -0.013491976074874401,...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2</td>
      <td>https://simple.wikipedia.org/wiki/August</td>
      <td>August</td>
      <td>August (Aug.) is the eighth month of the year ...</td>
      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>
      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>2</th>
      <td>6</td>
      <td>https://simple.wikipedia.org/wiki/Art</td>
      <td>Art</td>
      <td>Art is a creative activity that expresses imag...</td>
      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>
      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>3</th>
      <td>8</td>
      <td>https://simple.wikipedia.org/wiki/A</td>
      <td>A</td>
      <td>A or a is the first letter of the English alph...</td>
      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>
      <td>[0.024894846603274345, -0.022186409682035446, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>9</td>
      <td>https://simple.wikipedia.org/wiki/Air</td>
      <td>Air</td>
      <td>Air refers to the Earth's atmosphere.


