# embeddings_queue

created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 791739f6-ee4c-43d1-8674-117a97e03dd0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"791739f6-ee4c-43d1-8674-117a97e03dd0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cdae6e4c-62ff-4553-a080-f17895f59a8e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"40207fcc30b134bfaedcdd2ef0e30a170f8bcfbcc1f7e10f2b99a51adfd78857\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c6274e83-66c3-4a3c-8ba1-f7a82bce686d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"55d75d734c6c465942a4bbe99c9e2550ca1dde90d5af183d906f3fc23ae149c3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2288, \"end_char_idx\": 4729, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "```python\n# We'll need to install Qdrant client\n!pip install qdrant-client\n```\n\n\n```python\nimport openai\nimport pandas as pd\nfrom ast import literal_eval\nimport qdrant_client # Qdrant's client library for Python\n\n# This can be changed to the embedding model of your choice. Make sure its the same model that is used for generating embeddings\nEMBEDDING_MODEL = \"text-embedding-ada-002\"\n\n# Ignore unclosed SSL socket warnings - optional in case you get these errors\nimport warnings\n\nwarnings.filterwarnings(action=\"ignore\", message=\"unclosed\", category=ResourceWarning)\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n```\n\n## Load data\n\nIn this section we'll load embedded data that we've prepared previous to this session.\n\n\n```python\nimport requests\n\nembeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'\n\n# The file is ~700 MB so this will take some time\nresponse = requests.get(embeddings_url, verify=True)  # Set verify=False to bypass SSL verification\nwith open('vector_database_wikipedia_articles_embedded.zip', 'wb') as file:\n    file.write(response.content)\n```\n\n\n```python\nimport zipfile\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"../data\")\n```\n\n\n```python\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\n```\n\n\n```python\narticle_df.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.)"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: c6274e83-66c3-4a3c-8ba1-f7a82bce686d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c6274e83-66c3-4a3c-8ba1-f7a82bce686d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"791739f6-ee4c-43d1-8674-117a97e03dd0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3f4a3d28a02a799943d57f8daad30d55f975cea0fce0dd4ba960a3363ad27373\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a4b49e89-dc6e-4dfd-8776-e9311fdb7a53\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0eb9f1fbcad7c87e0ae1cc80d55f22c888212d66f0901dbcd88725f848899c3c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3582, \"end_char_idx\": 5760, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "```python\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\n```\n\n\n```python\narticle_df.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.) is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: a4b49e89-dc6e-4dfd-8776-e9311fdb7a53
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a4b49e89-dc6e-4dfd-8776-e9311fdb7a53\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c6274e83-66c3-4a3c-8ba1-f7a82bce686d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1143a72a567e41f711984a3dd61faf7b1acb703a4f0967eb68bfb1ea8a3d7583\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"93e4aebe-a03a-4be3-bc97-83aa09107041\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"28748822a10b93f32c3907b5ad7301f1dae153afc99238b87c700f33e40dc538\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4730, \"end_char_idx\": 6928, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere. Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\n# Read vectors from strings back into a list\narticle_df['title_vector'] = article_df.title_vector.apply(literal_eval)\narticle_df['content_vector'] = article_df.content_vector.apply(literal_eval)\n\n# Set vector_id to be a string\narticle_df['vector_id'] = article_df['vector_id'].apply(str)\n```\n\n\n```python\narticle_df.info(show_counts=True)\n```\n\n    <class 'pandas.core.frame.DataFrame'>\n    RangeIndex: 25000 entries, 0 to 24999\n    Data columns (total 7 columns):\n     #   Column          Non-Null Count  Dtype \n    ---  ------          --------------  ----- \n     0   id              25000 non-null  int64 \n     1   url             25000 non-null  object\n     2   title           25000 non-null  object\n     3   text            25000 non-null  object\n     4   title_vector    25000 non-null  object\n     5   content_vector  25000 non-null  object\n     6   vector_id       25000 non-null  object\n    dtypes: int64(1), object(6)\n    memory usage: 1.3+ MB"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 93e4aebe-a03a-4be3-bc97-83aa09107041
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"93e4aebe-a03a-4be3-bc97-83aa09107041\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a4b49e89-dc6e-4dfd-8776-e9311fdb7a53\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ec9019c56d4fb6c69725c130e71212e5c621b98ef2fa74fd4250c3bc63cf6a7d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1612bfcb-b2ee-43b8-b349-46af16d27e28\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"605efedf8157cc18d32721969f35ae03a1e32125dbe2e2c59aa1b214c6c3c8af\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5761, \"end_char_idx\": 8548, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\n# Read vectors from strings back into a list\narticle_df['title_vector'] = article_df.title_vector.apply(literal_eval)\narticle_df['content_vector'] = article_df.content_vector.apply(literal_eval)\n\n# Set vector_id to be a string\narticle_df['vector_id'] = article_df['vector_id'].apply(str)\n```\n\n\n```python\narticle_df.info(show_counts=True)\n```\n\n    <class 'pandas.core.frame.DataFrame'>\n    RangeIndex: 25000 entries, 0 to 24999\n    Data columns (total 7 columns):\n     #   Column          Non-Null Count  Dtype \n    ---  ------          --------------  ----- \n     0   id              25000 non-null  int64 \n     1   url             25000 non-null  object\n     2   title           25000 non-null  object\n     3   text            25000 non-null  object\n     4   title_vector    25000 non-null  object\n     5   content_vector  25000 non-null  object\n     6   vector_id       25000 non-null  object\n    dtypes: int64(1), object(6)\n    memory usage: 1.3+ MB\n\n\n## Qdrant\n\n**[Qdrant](https://qdrant.tech/)**. is a high-performant vector search database written in Rust. It offers both on-premise and cloud version, but for the purposes of that example we're going to use the local deployment mode.\n\nSetting everything up will require:\n- Spinning up a local instance of Qdrant\n- Configuring the collection and storing the data in it\n- Trying out with some queries\n\n### Setup\n\nFor the local deployment, we are going to use Docker, according to the Qdrant documentation: https://qdrant.tech/documentation/quick_start/. Qdrant requires just a single container, but an example of the docker-compose.yaml file is available at `./qdrant/docker-compose.yaml` in this repo.\n\nYou can start Qdrant instance locally by navigating to this directory and running `docker-compose up -d `\n\n> You might need to increase the memory limit for Docker to 8GB or more. Or Qdrant might fail to execute with an error message like `7 Killed`.\n\n\n\n```python\nqdrant = qdrant_client.QdrantClient(host=\"localhost\", port=6333)\n```\n\n\n```python\nqdrant.get_collections()\n```\n\n\n\n\n    CollectionsResponse(collections=[])\n\n\n\n### Index data\n\nQdrant stores data in __collections__ where each object is described by at least one vector and may contain an additional metadata called __payload__. Our collection will be called **Articles** and each object will be described by both **title** and **content** vectors.\n\nWe'll be using an official [qdrant-client](https://github.com/qdrant/qdrant_client) package that has all the utility methods already built-in.\n\n\n```python\nfrom qdrant_client.http import models as rest\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 1612bfcb-b2ee-43b8-b349-46af16d27e28
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1612bfcb-b2ee-43b8-b349-46af16d27e28\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"93e4aebe-a03a-4be3-bc97-83aa09107041\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e49f18ecd793b6e9310b06a40a21cc6e8922d33517a5a459a9dba38db82cb13e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1612308e-b878-4ff4-85f1-e1995a4384c8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6612104b42573d978d8386a61700753b3be566819ec4a60592f8a6473a34adcb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6931, \"end_char_idx\": 10105, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "## Qdrant\n\n**[Qdrant](https://qdrant.tech/)**. is a high-performant vector search database written in Rust. It offers both on-premise and cloud version, but for the purposes of that example we're going to use the local deployment mode.\n\nSetting everything up will require:\n- Spinning up a local instance of Qdrant\n- Configuring the collection and storing the data in it\n- Trying out with some queries\n\n### Setup\n\nFor the local deployment, we are going to use Docker, according to the Qdrant documentation: https://qdrant.tech/documentation/quick_start/. Qdrant requires just a single container, but an example of the docker-compose.yaml file is available at `./qdrant/docker-compose.yaml` in this repo.\n\nYou can start Qdrant instance locally by navigating to this directory and running `docker-compose up -d `\n\n> You might need to increase the memory limit for Docker to 8GB or more. Or Qdrant might fail to execute with an error message like `7 Killed`.\n\n\n\n```python\nqdrant = qdrant_client.QdrantClient(host=\"localhost\", port=6333)\n```\n\n\n```python\nqdrant.get_collections()\n```\n\n\n\n\n    CollectionsResponse(collections=[])\n\n\n\n### Index data\n\nQdrant stores data in __collections__ where each object is described by at least one vector and may contain an additional metadata called __payload__. Our collection will be called **Articles** and each object will be described by both **title** and **content** vectors.\n\nWe'll be using an official [qdrant-client](https://github.com/qdrant/qdrant_client) package that has all the utility methods already built-in.\n\n\n```python\nfrom qdrant_client.http import models as rest\n```\n\n\n```python\n# Get the vector size from the first row to set up the collection\nvector_size = len(article_df['content_vector'][0])\n\n# Set up the collection with the vector configuration. You need to declare the vector size and distance metric for the collection. Distance metric enables vector database to index and search vectors efficiently.\nqdrant.recreate_collection(\n    collection_name='Articles',\n    vectors_config={\n        'title': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n        'content': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n    }\n)\n```\n\n\n\n\n    True\n\n\n\n\n```python\nvector_size = len(article_df['content_vector'][0])\n\nqdrant.recreate_collection(\n    collection_name='Articles',\n    vectors_config={\n        'title': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n        'content': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n    }\n)\n```\n\n\n\n\n    True\n\n\n\nIn addition to the vector configuration defined under `vector`, we can also define the `payload` configuration. Payload is an optional field that allows you to store additional metadata alongside the vectors. In our case, we'll store the `id`, `title`, and `url` of the articles. As we return the title of nearest articles in the search results from payload, we can also provide the user with the URL to the article (which is part of the meta-data)."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 1612308e-b878-4ff4-85f1-e1995a4384c8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1612308e-b878-4ff4-85f1-e1995a4384c8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1612bfcb-b2ee-43b8-b349-46af16d27e28\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"5291e5704a1db8a66a1ea6ec09f261bd3cecf3669370c8d823b13705506541e9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2df4ccb1-964b-415c-90f2-dc5c6851227a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"562ad1d5064f5bb48f1e321f2b3896e8bacf31c6da25f9e25f1d241cf88e8b62\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8489, \"end_char_idx\": 11487, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "```python\nfrom qdrant_client.http import models as rest\n```\n\n\n```python\n# Get the vector size from the first row to set up the collection\nvector_size = len(article_df['content_vector'][0])\n\n# Set up the collection with the vector configuration. You need to declare the vector size and distance metric for the collection. Distance metric enables vector database to index and search vectors efficiently.\nqdrant.recreate_collection(\n    collection_name='Articles',\n    vectors_config={\n        'title': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n        'content': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n    }\n)\n```\n\n\n\n\n    True\n\n\n\n\n```python\nvector_size = len(article_df['content_vector'][0])\n\nqdrant.recreate_collection(\n    collection_name='Articles',\n    vectors_config={\n        'title': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n        'content': rest.VectorParams(\n            distance=rest.Distance.COSINE,\n            size=vector_size,\n        ),\n    }\n)\n```\n\n\n\n\n    True\n\n\n\nIn addition to the vector configuration defined under `vector`, we can also define the `payload` configuration. Payload is an optional field that allows you to store additional metadata alongside the vectors. In our case, we'll store the `id`, `title`, and `url` of the articles. As we return the title of nearest articles in the search results from payload, we can also provide the user with the URL to the article (which is part of the meta-data).\n\n\n```python\nfrom qdrant_client.models import PointStruct # Import the PointStruct to store the vector and payload\nfrom tqdm import tqdm # Library to show the progress bar \n\n# Populate collection with vectors using tqdm to show progress\nfor k, v in tqdm(article_df.iterrows(), desc=\"Upserting articles\", total=len(article_df)):\n    try:\n        qdrant.upsert(\n            collection_name='Articles',\n            points=[\n                PointStruct(\n                    id=k,\n                    vector={'title': v['title_vector'], \n                            'content': v['content_vector']},\n                    payload={\n                        'id': v['id'],\n                        'title': v['title'],\n                        'url': v['url']\n                    }\n                )\n            ]\n        )\n    except Exception as e:\n        print(f\"Failed to upsert row {k}: {v}\")\n        print(f\"Exception: {e}\")\n```\n\n    Upserting articles: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25000/25000 [01:34<00:00, 264.52it/s]\n\n\n\n```python\n# Check the collection size to make sure all the points have been stored\nqdrant.count(collection_name='Articles')\n```\n\n\n\n\n    CountResult(count=25000)\n\n\n\n### Search Data\n\nOnce the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 2df4ccb1-964b-415c-90f2-dc5c6851227a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2df4ccb1-964b-415c-90f2-dc5c6851227a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1612308e-b878-4ff4-85f1-e1995a4384c8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1413e53adcbb5bf1f4adfebdbbe92f56813991134f11994e6c8cee1790e46533\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7781d77c-050a-49c0-a4fe-04ef57214996\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"80c62146f4167613ab12d663ab8737bbfa3a8784b187b27f853d4e8984a40fa5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9656, \"end_char_idx\": 12030, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "In addition to the vector configuration defined under `vector`, we can also define the `payload` configuration. Payload is an optional field that allows you to store additional metadata alongside the vectors. In our case, we'll store the `id`, `title`, and `url` of the articles. As we return the title of nearest articles in the search results from payload, we can also provide the user with the URL to the article (which is part of the meta-data).\n\n\n```python\nfrom qdrant_client.models import PointStruct # Import the PointStruct to store the vector and payload\nfrom tqdm import tqdm # Library to show the progress bar \n\n# Populate collection with vectors using tqdm to show progress\nfor k, v in tqdm(article_df.iterrows(), desc=\"Upserting articles\", total=len(article_df)):\n    try:\n        qdrant.upsert(\n            collection_name='Articles',\n            points=[\n                PointStruct(\n                    id=k,\n                    vector={'title': v['title_vector'], \n                            'content': v['content_vector']},\n                    payload={\n                        'id': v['id'],\n                        'title': v['title'],\n                        'url': v['url']\n                    }\n                )\n            ]\n        )\n    except Exception as e:\n        print(f\"Failed to upsert row {k}: {v}\")\n        print(f\"Exception: {e}\")\n```\n\n    Upserting articles: 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 25000/25000 [01:34<00:00, 264.52it/s]\n\n\n\n```python\n# Check the collection size to make sure all the points have been stored\nqdrant.count(collection_name='Articles')\n```\n\n\n\n\n    CountResult(count=25000)\n\n\n\n### Search Data\n\nOnce the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search.\n\n\n```python\ndef query_qdrant(query, collection_name, vector_name='title', top_k=20):\n\n    # Creates embedding vector from user query\n    embedded_query = openai.embeddings.create(\n        input=query,\n        model=EMBEDDING_MODEL,\n    ).data[0].embedding # We take the first embedding from the list\n    \n    query_results = qdrant.search(\n        collection_name=collection_name,\n        query_vector=(\n            vector_name, embedded_query\n        ),\n        limit=top_k, \n        query_filter=None\n    )\n    \n    return query_results\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 7781d77c-050a-49c0-a4fe-04ef57214996
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7781d77c-050a-49c0-a4fe-04ef57214996\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2df4ccb1-964b-415c-90f2-dc5c6851227a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e12fffe0c2876d962a408b63d563f9d92e04e317321887d58e88fe17ba48702b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0272a28d-33ef-4b2a-ab5a-e53f12adffa1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"517a83f991644f2cbc2ea6b6838ef3e32d8a19c57de88defc02f74e717c86e02\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11112, \"end_char_idx\": 14064, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "```python\n# Check the collection size to make sure all the points have been stored\nqdrant.count(collection_name='Articles')\n```\n\n\n\n\n    CountResult(count=25000)\n\n\n\n### Search Data\n\nOnce the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search.\n\n\n```python\ndef query_qdrant(query, collection_name, vector_name='title', top_k=20):\n\n    # Creates embedding vector from user query\n    embedded_query = openai.embeddings.create(\n        input=query,\n        model=EMBEDDING_MODEL,\n    ).data[0].embedding # We take the first embedding from the list\n    \n    query_results = qdrant.search(\n        collection_name=collection_name,\n        query_vector=(\n            vector_name, embedded_query\n        ),\n        limit=top_k, \n        query_filter=None\n    )\n    \n    return query_results\n```\n\n\n```python\nquery_results = query_qdrant('modern art in Europe', 'Articles', 'title')\nfor i, article in enumerate(query_results):\n    print(f'{i + 1}. {article.payload[\"title\"]}, URL: {article.payload['url']} (Score: {round(article.score, 3)})')\n```\n\n    1. Museum of Modern Art, URL: https://simple.wikipedia.org/wiki/Museum%20of%20Modern%20Art (Score: 0.875)\n    2. Western Europe, URL: https://simple.wikipedia.org/wiki/Western%20Europe (Score: 0.867)\n    3. Renaissance art, URL: https://simple.wikipedia.org/wiki/Renaissance%20art (Score: 0.864)\n    4. Pop art, URL: https://simple.wikipedia.org/wiki/Pop%20art (Score: 0.86)\n    5. Northern Europe, URL: https://simple.wikipedia.org/wiki/Northern%20Europe (Score: 0.855)\n    6. Hellenistic art, URL: https://simple.wikipedia.org/wiki/Hellenistic%20art (Score: 0.853)\n    7. Modernist literature, URL: https://simple.wikipedia.org/wiki/Modernist%20literature (Score: 0.847)\n    8. Art film, URL: https://simple.wikipedia.org/wiki/Art%20film (Score: 0.843)\n    9. Central Europe, URL: https://simple.wikipedia.org/wiki/Central%20Europe (Score: 0.842)\n    10. European, URL: https://simple.wikipedia.org/wiki/European (Score: 0.841)\n    11. Art, URL: https://simple.wikipedia.org/wiki/Art (Score: 0.841)\n    12. Byzantine art, URL: https://simple.wikipedia.org/wiki/Byzantine%20art (Score: 0.841)\n    13. Postmodernism, URL: https://simple.wikipedia.org/wiki/Postmodernism (Score: 0.84)\n    14. Eastern Europe, URL: https://simple.wikipedia.org/wiki/Eastern%20Europe (Score: 0.839)\n    15. Cubism, URL: https://simple.wikipedia.org/wiki/Cubism (Score: 0.839)\n    16. Europe, URL: https://simple.wikipedia.org/wiki/Europe (Score: 0.839)\n    17. Impressionism, URL: https://simple.wikipedia.org/wiki/Impressionism (Score: 0.838)\n    18. Bauhaus, URL: https://simple.wikipedia.org/wiki/Bauhaus (Score: 0.838)\n    19. Surrealism, URL: https://simple.wikipedia.org/wiki/Surrealism (Score: 0.837)\n    20. Expressionism, URL: https://simple.wikipedia.org/wiki/Expressionism (Score: 0.837)"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 0272a28d-33ef-4b2a-ab5a-e53f12adffa1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb", "title": "Using Qdrant for Embeddings Search", "tokens": 4309, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0272a28d-33ef-4b2a-ab5a-e53f12adffa1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0a5e2d7a-39ac-470a-86ab-4a93d4efd483\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"60898b70a600f0a7e8231cd43e833d8fb18d7a9b2db28edcadf9249732cad1c8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7781d77c-050a-49c0-a4fe-04ef57214996\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/qdrant/Using_Qdrant_for_embeddings_search.ipynb\", \"title\": \"Using Qdrant for Embeddings Search\", \"tokens\": 4309, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a112b5ad9678b75d5d61f6c05bf9d5e9a247b783bc62e7d84b3f3ce5cd5e4b0e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14068, \"end_char_idx\": 16134, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "ref_doc_id": "0a5e2d7a-39ac-470a-86ab-4a93d4efd483", "chroma:document": "```python\n# This time we'll query using content vector\nquery_results = query_qdrant('Famous battles in Scottish history', 'Articles', 'content')\nfor i, article in enumerate(query_results):\n    print(f'{i + 1}. {article.payload[\"title\"]}, URL: {article.payload['url']} (Score: {round(article.score, 3)})')\n```\n\n    1. Battle of Bannockburn, URL: https://simple.wikipedia.org/wiki/Battle%20of%20Bannockburn (Score: 0.869)\n    2. Wars of Scottish Independence, URL: https://simple.wikipedia.org/wiki/Wars%20of%20Scottish%20Independence (Score: 0.861)\n    3. 1651, URL: https://simple.wikipedia.org/wiki/1651 (Score: 0.852)\n    4. First War of Scottish Independence, URL: https://simple.wikipedia.org/wiki/First%20War%20of%20Scottish%20Independence (Score: 0.85)\n    5. Robert I of Scotland, URL: https://simple.wikipedia.org/wiki/Robert%20I%20of%20Scotland (Score: 0.846)\n    6. 841, URL: https://simple.wikipedia.org/wiki/841 (Score: 0.844)\n    7. 1716, URL: https://simple.wikipedia.org/wiki/1716 (Score: 0.844)\n    8. 1314, URL: https://simple.wikipedia.org/wiki/1314 (Score: 0.837)\n    9. 1263, URL: https://simple.wikipedia.org/wiki/1263 (Score: 0.836)\n    10. William Wallace, URL: https://simple.wikipedia.org/wiki/William%20Wallace (Score: 0.835)\n    11. Stirling, URL: https://simple.wikipedia.org/wiki/Stirling (Score: 0.831)\n    12. 1306, URL: https://simple.wikipedia.org/wiki/1306 (Score: 0.831)\n    13. 1746, URL: https://simple.wikipedia.org/wiki/1746 (Score: 0.83)\n    14. 1040s, URL: https://simple.wikipedia.org/wiki/1040s (Score: 0.828)\n    15. 1106, URL: https://simple.wikipedia.org/wiki/1106 (Score: 0.827)\n    16. 1304, URL: https://simple.wikipedia.org/wiki/1304 (Score: 0.826)\n    17. David II of Scotland, URL: https://simple.wikipedia.org/wiki/David%20II%20of%20Scotland (Score: 0.825)\n    18. Braveheart, URL: https://simple.wikipedia.org/wiki/Braveheart (Score: 0.824)\n    19. 1124, URL: https://simple.wikipedia.org/wiki/1124 (Score: 0.824)\n    20. July 27, URL: https://simple.wikipedia.org/wiki/July%2027 (Score: 0.823)\n\n\n\n```python\n\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: cf9adf32-4eac-4add-99de-7011cfe2c783
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"cf9adf32-4eac-4add-99de-7011cfe2c783\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"827f149c-1e78-4129-b1f3-ab03a247234b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ab180f4fa0a8c795bf3f03b7a1b348cc1d2278e23074955f1906bcabcaf2719a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2845, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "# Philosophy with Vector Embeddings, OpenAI and Astra DB\n\n### AstraPy version\n\nIn this quickstart you will learn how to build a \"philosophy quote finder & generator\" using OpenAI's vector embeddings and DataStax [Astra DB](https://docs.datastax.com/en/astra/home/astra.html) as the vector store for data persistence.\n\nThe basic workflow of this notebook is outlined below. You will evaluate and store the vector embeddings for a number of quotes by famous philosophers, use them to build a powerful search engine and, after that, even a generator of new quotes!\n\nThe notebook exemplifies some of the standard usage patterns of vector search -- while showing how easy is it to get started with [Astra DB](https://docs.datastax.com/en/astra/home/astra.html).\n\nFor a background on using vector search and text embeddings to build a question-answering system, please check out this excellent hands-on notebook: [Question answering using embeddings](https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb).\n\nTable of contents:\n- Setup\n- Create vector collection\n- Connect to OpenAI\n- Load quotes into the Vector Store\n- Use case 1: **quote search engine**\n- Use case 2: **quote generator**\n- Cleanup\n\n### How it works\n\n**Indexing**\n\nEach quote is made into an embedding vector with OpenAI's `Embedding`. These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing](https://user-images.githubusercontent.com/14221764/282422016-1d540607-eed4-4240-9c3d-22ee3a3bc90f.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search](https://user-images.githubusercontent.com/14221764/282422033-0a1297c4-63bb-4e04-b120-dfd98dc1a689.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n!"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 827f149c-1e78-4129-b1f3-ab03a247234b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"827f149c-1e78-4129-b1f3-ab03a247234b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cf9adf32-4eac-4add-99de-7011cfe2c783\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"bb310b4940bde621974587766d02ee9a957146f9595860205f2358bc4c46d0c3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"17e379c9-340a-43f6-805a-ccb87f9454bc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"29a7c8c0dbc05c6cff49691bebc6f9c06ca66312a8c27582e64dba2b8dbb6336\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1347, \"end_char_idx\": 4009, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing](https://user-images.githubusercontent.com/14221764/282422016-1d540607-eed4-4240-9c3d-22ee3a3bc90f.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search](https://user-images.githubusercontent.com/14221764/282422033-0a1297c4-63bb-4e04-b120-dfd98dc1a689.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282422050-2e209ff5-07d6-41ac-99ac-f442e090b3bb.png)\n\n## Setup\n\nInstall and import the necessary dependencies:\n\n\n```python\n!pip install --quiet \"astrapy>=0.6.0\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nfrom getpass import getpass\nfrom collections import Counter\n\nfrom astrapy.db import AstraDB\nimport openai\nfrom datasets import load_dataset\n```\n\n### Connection parameters\n\nPlease retrieve your database credentials on your Astra dashboard ([info](https://docs.datastax.com/en/astra/astra-db-vector/)): you will supply them momentarily.\n\nExample values:\n\n- API Endpoint: `https://01234567-89ab-cdef-0123-456789abcdef-us-east1.apps.astra.datastax.com`\n- Token: `AstraCS:6gBhNmsk135...`"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 17e379c9-340a-43f6-805a-ccb87f9454bc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"17e379c9-340a-43f6-805a-ccb87f9454bc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"827f149c-1e78-4129-b1f3-ab03a247234b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1f966e5e01147515f1c416715f7c66bbb76c537476196b73f48376bf55d6aa62\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"452b1cd9-d70d-47d1-a09c-dde70e222c21\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0ce899b54b844a5aa2e799d742f1933cfe8fc4a1a972e28260739b83f34b39d0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2533, \"end_char_idx\": 5091, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282422050-2e209ff5-07d6-41ac-99ac-f442e090b3bb.png)\n\n## Setup\n\nInstall and import the necessary dependencies:\n\n\n```python\n!pip install --quiet \"astrapy>=0.6.0\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nfrom getpass import getpass\nfrom collections import Counter\n\nfrom astrapy.db import AstraDB\nimport openai\nfrom datasets import load_dataset\n```\n\n### Connection parameters\n\nPlease retrieve your database credentials on your Astra dashboard ([info](https://docs.datastax.com/en/astra/astra-db-vector/)): you will supply them momentarily.\n\nExample values:\n\n- API Endpoint: `https://01234567-89ab-cdef-0123-456789abcdef-us-east1.apps.astra.datastax.com`\n- Token: `AstraCS:6gBhNmsk135...`\n\n\n```python\nASTRA_DB_API_ENDPOINT = input(\"Please enter your API Endpoint:\")\nASTRA_DB_APPLICATION_TOKEN = getpass(\"Please enter your Token\")\n```\n\n    Please enter your API Endpoint: https://4f835778-ec78-42b0-9ae3-29e3cf45b596-us-east1.apps.astra.datastax.com\n    Please enter your Token \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### Instantiate an Astra DB client\n\n\n```python\nastra_db = AstraDB(\n    api_endpoint=ASTRA_DB_API_ENDPOINT,\n    token=ASTRA_DB_APPLICATION_TOKEN,\n)\n```\n\n## Create vector collection\n\nThe only parameter to specify, other than the collection name, is the dimension of the vectors you'll store. Other parameters, notably the similarity metric to use for searches, are optional.\n\n\n```python\ncoll_name = \"philosophers_astra_db\"\ncollection = astra_db.create_collection(coll_name, dimension=1536)\n```\n\n## Connect to OpenAI\n\n### Set up your secret key\n\n\n```python\nOPENAI_API_KEY = getpass(\"Please enter your OpenAI API Key: \")\n```\n\n    Please enter your OpenAI API Key:  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 452b1cd9-d70d-47d1-a09c-dde70e222c21
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"452b1cd9-d70d-47d1-a09c-dde70e222c21\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"17e379c9-340a-43f6-805a-ccb87f9454bc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2659e9bcbf8755c65da3845a34be54f98020795817908cb9fd735978520ebf13\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"85e75e75-2284-4041-94b3-a345ef87ee37\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"169441b9efdee16ad99ddbc10eee6e264af7c0a4008eaa31ed4c43c54dd009b6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4012, \"end_char_idx\": 6454, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\nASTRA_DB_API_ENDPOINT = input(\"Please enter your API Endpoint:\")\nASTRA_DB_APPLICATION_TOKEN = getpass(\"Please enter your Token\")\n```\n\n    Please enter your API Endpoint: https://4f835778-ec78-42b0-9ae3-29e3cf45b596-us-east1.apps.astra.datastax.com\n    Please enter your Token \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### Instantiate an Astra DB client\n\n\n```python\nastra_db = AstraDB(\n    api_endpoint=ASTRA_DB_API_ENDPOINT,\n    token=ASTRA_DB_APPLICATION_TOKEN,\n)\n```\n\n## Create vector collection\n\nThe only parameter to specify, other than the collection name, is the dimension of the vectors you'll store. Other parameters, notably the similarity metric to use for searches, are optional.\n\n\n```python\ncoll_name = \"philosophers_astra_db\"\ncollection = astra_db.create_collection(coll_name, dimension=1536)\n```\n\n## Connect to OpenAI\n\n### Set up your secret key\n\n\n```python\nOPENAI_API_KEY = getpass(\"Please enter your OpenAI API Key: \")\n```\n\n    Please enter your OpenAI API Key:  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nprint(f\"len(result.data)              = {len(result.data)}\")\nprint(f\"result.data[1].embedding      = {str(result.data[1].embedding)[:55]}...\")\nprint(f\"len(result.data[1].embedding) = {len(result.data[1].embedding)}\")\n```\n\n    len(result.data)              = 2\n    result.data[1].embedding      = [-0.0108176339417696, 0.0013546717818826437, 0.00362232...\n    len(result.data[1].embedding) = 1536\n\n\n## Load quotes into the Vector Store\n\nGet a dataset with the quotes. _(We adapted and augmented the data from [this Kaggle dataset](https://www.kaggle.com/datasets/mertbozkurt5/quotes-by-philosophers), ready to use in this demo.)_\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 85e75e75-2284-4041-94b3-a345ef87ee37
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"85e75e75-2284-4041-94b3-a345ef87ee37\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"452b1cd9-d70d-47d1-a09c-dde70e222c21\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"0f5e38bbfdea66c52971ec9216b1270c384121f5ed90ee47c16fb23dd7e9ab44\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5221699d-6df0-4d9d-b2b0-cd67844ed445\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"854dbf941b3f333d41e0c2f2e89d75b56943c35422aaee4d6517978e4030a52e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4981, \"end_char_idx\": 7654, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nprint(f\"len(result.data)              = {len(result.data)}\")\nprint(f\"result.data[1].embedding      = {str(result.data[1].embedding)[:55]}...\")\nprint(f\"len(result.data[1].embedding) = {len(result.data[1].embedding)}\")\n```\n\n    len(result.data)              = 2\n    result.data[1].embedding      = [-0.0108176339417696, 0.0013546717818826437, 0.00362232...\n    len(result.data[1].embedding) = 1536\n\n\n## Load quotes into the Vector Store\n\nGet a dataset with the quotes. _(We adapted and augmented the data from [this Kaggle dataset](https://www.kaggle.com/datasets/mertbozkurt5/quotes-by-philosophers), ready to use in this demo.)_\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Write to the vector collection\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata you'll use later.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\nTo store the quote objects, you will use the `insert_many` method of the collection (one call per batch). When preparing the documents for insertion you will choose suitable field names -- keep in mind, however, that the embedding vector must be the fixed special `$vector` field."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 5221699d-6df0-4d9d-b2b0-cd67844ed445
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5221699d-6df0-4d9d-b2b0-cd67844ed445\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"85e75e75-2284-4041-94b3-a345ef87ee37\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"08b1bcbe26c4d3a2826fc02755ff59631c92438f31c58251f759ffef009f77d7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"98ea0372-20ad-4cb5-b604-bdb40c1d8ca9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f0e52235ed9aaab6d406c85b38d226f19fdd457265c29ef37083fac930d16e65\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6124, \"end_char_idx\": 9033, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Write to the vector collection\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata you'll use later.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\nTo store the quote objects, you will use the `insert_many` method of the collection (one call per batch). When preparing the documents for insertion you will choose suitable field names -- keep in mind, however, that the embedding vector must be the fixed special `$vector` field.\n\n\n```python\nBATCH_SIZE = 20\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries: \", end=\"\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the documents for insertion\n    b_docs = []\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag: True\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = {}\n        b_docs.append({\n            \"quote\": quotes_list[entry_idx],\n            \"$vector\": emb_result.embedding,\n            \"author\": authors_list[entry_idx],\n            \"tags\": tags,\n        })\n    # write to the vector collection\n    collection.insert_many(b_docs)\n    print(f\"[{len(b_docs)}]\", end=\"\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries: [20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][10]\n    Finished storing entries."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 98ea0372-20ad-4cb5-b604-bdb40c1d8ca9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"98ea0372-20ad-4cb5-b604-bdb40c1d8ca9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5221699d-6df0-4d9d-b2b0-cd67844ed445\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3b6b1dcaa389277ed13f0385d356299965b78aa29e7bb0889d249fbb11b697e5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2f1aa9db-2006-4691-afde-86364493059a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"200cc493de808d61f5b2b3cfb6ba74fdcd35a0334001a15f19fc5109b60bdab5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7657, \"end_char_idx\": 10815, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\nBATCH_SIZE = 20\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries: \", end=\"\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the documents for insertion\n    b_docs = []\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag: True\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = {}\n        b_docs.append({\n            \"quote\": quotes_list[entry_idx],\n            \"$vector\": emb_result.embedding,\n            \"author\": authors_list[entry_idx],\n            \"tags\": tags,\n        })\n    # write to the vector collection\n    collection.insert_many(b_docs)\n    print(f\"[{len(b_docs)}]\", end=\"\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries: [20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][20][10]\n    Finished storing entries.\n\n\n## Use case 1: **quote search engine**\n\nFor the quote-search functionality, you need first to make the input quote into a vector, and then use it to query the store (besides handling the optional metadata into the search call, that is).\n\nEncapsulate the search-engine functionality into a function for ease of re-use. At its core is the `vector_find` method of the collection:\n\n\n```python\ndef find_quote_and_author(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    filter_clause = {}\n    if author:\n        filter_clause[\"author\"] = author\n    if tags:\n        filter_clause[\"tags\"] = {}\n        for tag in tags:\n            filter_clause[\"tags\"][tag] = True\n    #\n    results = collection.vector_find(\n        query_vector,\n        limit=n,\n        filter=filter_clause,\n        fields=[\"quote\", \"author\"]\n    )\n    return [\n        (result[\"quote\"], result[\"author\"])\n        for result in results\n    ]\n```\n\n### Putting search to test\n\nPassing just a quote:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 2f1aa9db-2006-4691-afde-86364493059a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2f1aa9db-2006-4691-afde-86364493059a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"98ea0372-20ad-4cb5-b604-bdb40c1d8ca9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"180f4cc35f952f8f8599bb203c13a7acf455f6c11fabc7caa62d5a6fbd28e90b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e6d59c0e-921e-4ccc-9214-894e6ccb05a7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e9bac3a77834727f2424f18bfaf5a527f17c8d8e545a159a1b262d3c24ab7e5c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9415, \"end_char_idx\": 12809, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\ndef find_quote_and_author(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    filter_clause = {}\n    if author:\n        filter_clause[\"author\"] = author\n    if tags:\n        filter_clause[\"tags\"] = {}\n        for tag in tags:\n            filter_clause[\"tags\"][tag] = True\n    #\n    results = collection.vector_find(\n        query_vector,\n        limit=n,\n        filter=filter_clause,\n        fields=[\"quote\", \"author\"]\n    )\n    return [\n        (result[\"quote\"], result[\"author\"])\n        for result in results\n    ]\n```\n\n### Putting search to test\n\nPassing just a quote:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nSearch constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('He who seeks equality between unequals seeks an absurdity.', 'spinoza'),\n     ('The people are that part of the state that does not know what it wants.',\n      'hegel')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"similarity\" between the query and each result, and then implement a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results. Note that the similarity is returned as the special `$similarity` field in each result document - and it will be returned by default, unless you pass `include_similarity = False` to the search method.\n\n_Note (for the mathematically inclined): this value is **a rescaling between zero and one** of the cosine difference between the vectors, i.e. of the scalar product divided by the product of the norms of the two vectors. In other words, this is 0 for opposite-facing vectors and +1 for parallel vectors. For other measures of similarity (cosine is the default), check the `metric` parameter in `AstraDB.create_collection` and the [documentation on allowed values](https://docs.datastax.com/en/astra-serverless/docs/develop/dev-with-json.html#metric-types)._"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: e6d59c0e-921e-4ccc-9214-894e6ccb05a7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e6d59c0e-921e-4ccc-9214-894e6ccb05a7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2f1aa9db-2006-4691-afde-86364493059a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"432dd9819d4fb006a774e9be398fa31ffaac798be99f2f726411c48119c0f9d3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3c336144-1ee1-42b9-a108-d83a5fb1820b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f90b576f53a61e402f47adb759bc54571029d3af9550cc33d991be8c8c56ef71\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11050, \"end_char_idx\": 14070, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "Search constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('He who seeks equality between unequals seeks an absurdity.', 'spinoza'),\n     ('The people are that part of the state that does not know what it wants.',\n      'hegel')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"similarity\" between the query and each result, and then implement a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results. Note that the similarity is returned as the special `$similarity` field in each result document - and it will be returned by default, unless you pass `include_similarity = False` to the search method.\n\n_Note (for the mathematically inclined): this value is **a rescaling between zero and one** of the cosine difference between the vectors, i.e. of the scalar product divided by the product of the norms of the two vectors. In other words, this is 0 for opposite-facing vectors and +1 for parallel vectors. For other measures of similarity (cosine is the default), check the `metric` parameter in `AstraDB.create_collection` and the [documentation on allowed values](https://docs.datastax.com/en/astra-serverless/docs/develop/dev-with-json.html#metric-types)._\n\n\n```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nmetric_threshold = 0.92\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\nresults_full = collection.vector_find(\n    quote_vector,\n    limit=8,\n    fields=[\"quote\"]\n)\nresults = [res for res in results_full if res[\"$similarity\"] >= metric_threshold]\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, result in enumerate(results):\n    print(f\"    {idx}. [similarity={result['$similarity']:.3f}] \\\"{result['quote'][:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [similarity=0.927] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [similarity=0.922] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [similarity=0.920] \"At his best, man is the noblest of all animals; separated from law and...\"\n\n\n## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 3c336144-1ee1-42b9-a108-d83a5fb1820b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3c336144-1ee1-42b9-a108-d83a5fb1820b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e6d59c0e-921e-4ccc-9214-894e6ccb05a7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"5fbf00135d71e7ef4668250fd60e3cd6d8cea0b35ce4c2f745dac24360f8b8be\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"70825f75-d48a-466f-a155-24fa91caaf03\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"87d649dcc77f63aabadcd8955cdbafc2e494ca55c06478ba4ce43b4092fd36dc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12812, \"end_char_idx\": 15716, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nmetric_threshold = 0.92\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\nresults_full = collection.vector_find(\n    quote_vector,\n    limit=8,\n    fields=[\"quote\"]\n)\nresults = [res for res in results_full if res[\"$similarity\"] >= metric_threshold]\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, result in enumerate(results):\n    print(f\"    {idx}. [similarity={result['$similarity']:.3f}] \\\"{result['quote'][:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [similarity=0.927] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [similarity=0.922] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [similarity=0.920] \"At his best, man is the noblest of all animals; separated from law and...\"\n\n\n## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task.\n\n\n```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):\n\n\n```python\ndef generate_quote(topic, n=2, author=None, tags=None):\n    quotes = find_quote_and_author(query_quote=topic, n=n, author=author, tags=tags)\n    if quotes:\n        prompt = generation_prompt_template.format(\n            topic=topic,\n            examples=\"\\n\".join(f\"  - {quote[0]}\" for quote in quotes),\n        )\n        # a little logging:\n        print(\"** quotes found:\")\n        for q, a in quotes:\n            print(f\"**    - {q} ({a})\")\n        print(\"** end of logging\")\n        #\n        response = client.chat.completions.create(\n            model=completion_model_name,\n            messages=[{\"role\": \"user\", \"content\": prompt}],\n            temperature=0.7,\n            max_tokens=320,\n        )\n        return response.choices[0].message.content.replace('\"', '').strip()\n    else:\n        print(\"** no quotes found.\")\n        return None\n```\n\n_Note: similar to the case of the embedding computation, the code for the Chat Completion API would be slightly different for OpenAI prior to v1.0._\n\n#### Putting quote generation to test\n\nJust passing a text (a \"quote\", but one can actually just suggest a topic since its vector embedding will still end up at the right place in the vector space):"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 70825f75-d48a-466f-a155-24fa91caaf03
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"70825f75-d48a-466f-a155-24fa91caaf03\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3c336144-1ee1-42b9-a108-d83a5fb1820b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a8d7bfb6d4a3baadc8848d8022e1d89507a7f3c5b08a345c9a7d631db51eeb94\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c384c9da-fe42-4a91-a8c4-642153ff42ed\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3e1b18dd4edc7b304f7549d0b75288c20dde2dab2b067974e6388fb9dfcb23c4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14073, \"end_char_idx\": 17573, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):\n\n\n```python\ndef generate_quote(topic, n=2, author=None, tags=None):\n    quotes = find_quote_and_author(query_quote=topic, n=n, author=author, tags=tags)\n    if quotes:\n        prompt = generation_prompt_template.format(\n            topic=topic,\n            examples=\"\\n\".join(f\"  - {quote[0]}\" for quote in quotes),\n        )\n        # a little logging:\n        print(\"** quotes found:\")\n        for q, a in quotes:\n            print(f\"**    - {q} ({a})\")\n        print(\"** end of logging\")\n        #\n        response = client.chat.completions.create(\n            model=completion_model_name,\n            messages=[{\"role\": \"user\", \"content\": prompt}],\n            temperature=0.7,\n            max_tokens=320,\n        )\n        return response.choices[0].message.content.replace('\"', '').strip()\n    else:\n        print(\"** no quotes found.\")\n        return None\n```\n\n_Note: similar to the case of the embedding computation, the code for the Chat Completion API would be slightly different for OpenAI prior to v1.0._\n\n#### Putting quote generation to test\n\nJust passing a text (a \"quote\", but one can actually just suggest a topic since its vector embedding will still end up at the right place in the vector space):\n\n\n```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    True politics lies in the virtuous pursuit of justice, for it is through virtue that we build a better world for all.\n\n\nUse inspiration from just a single philosopher:\n\n\n```python\nq_topic = generate_quote(\"animals\", author=\"schopenhauer\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Because Christian morality leaves animals out of account, they are at once outlawed in philosophical morals; they are mere 'things,' mere means to any ends whatsoever. They can therefore be used for vivisection, hunting, coursing, bullfights, and horse racing, and can be whipped to death as they struggle along with heavy carts of stone. Shame on such a morality that is worthy of pariahs, and that fails to recognize the eternal essence that exists in every living thing, and shines forth with inscrutable significance from all eyes that see the sun! (schopenhauer)\n    **    - The assumption that animals are without rights, and the illusion that our treatment of them has no moral significance, is a positively outrageous example of Western crudity and barbarity. Universal compassion is the only guarantee of morality. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    Excluding animals from ethical consideration reveals a moral blindness that allows for their exploitation and suffering. True morality embraces universal compassion."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: c384c9da-fe42-4a91-a8c4-642153ff42ed
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Astra DB", "tokens": 4213, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c384c9da-fe42-4a91-a8c4-642153ff42ed\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"ca3153cb-e17c-4090-97a6-dc4ba3a05a59\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"11ecc2b89c2290c6c16b28186eb94888f10182554adae700c5a4235db2a57141\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"70825f75-d48a-466f-a155-24fa91caaf03\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Astra DB\", \"tokens\": 4213, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2f2b0c1eef644bdac0b5ba8eece7b4ebe6e954b8d7eea624d0e72b30fa4d39e8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15719, \"end_char_idx\": 17814, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "ref_doc_id": "ca3153cb-e17c-4090-97a6-dc4ba3a05a59", "chroma:document": "```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    True politics lies in the virtuous pursuit of justice, for it is through virtue that we build a better world for all.\n\n\nUse inspiration from just a single philosopher:\n\n\n```python\nq_topic = generate_quote(\"animals\", author=\"schopenhauer\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Because Christian morality leaves animals out of account, they are at once outlawed in philosophical morals; they are mere 'things,' mere means to any ends whatsoever. They can therefore be used for vivisection, hunting, coursing, bullfights, and horse racing, and can be whipped to death as they struggle along with heavy carts of stone. Shame on such a morality that is worthy of pariahs, and that fails to recognize the eternal essence that exists in every living thing, and shines forth with inscrutable significance from all eyes that see the sun! (schopenhauer)\n    **    - The assumption that animals are without rights, and the illusion that our treatment of them has no moral significance, is a positively outrageous example of Western crudity and barbarity. Universal compassion is the only guarantee of morality. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    Excluding animals from ethical consideration reveals a moral blindness that allows for their exploitation and suffering. True morality embraces universal compassion.\n\n\n## Cleanup\n\nIf you want to remove all resources used for this demo, run this cell (_warning: this will irreversibly delete the collection and its data!_):\n\n\n```python\nastra_db.delete_collection(coll_name)\n```\n\n\n\n\n    {'status': {'ok': 1}}"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 6f5e4f4a-911d-48d0-916d-8133c329c1e3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6f5e4f4a-911d-48d0-916d-8133c329c1e3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d8ae302a-0e61-4c13-a635-2ceb44c39f2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"18f245ededb9bdabd0c5caae93afbc75bcf293d33731a39326b9e92ef3742d16\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3309, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "# Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\n\n### CQL Version\n\nIn this quickstart you will learn how to build a \"philosophy quote finder & generator\" using OpenAI's vector embeddings and [Apache Cassandra\u00ae](https://cassandra.apache.org), or equivalently DataStax [Astra DB through CQL](https://docs.datastax.com/en/astra-serverless/docs/vector-search/quickstart.html), as the vector store for data persistence.\n\nThe basic workflow of this notebook is outlined below. You will evaluate and store the vector embeddings for a number of quotes by famous philosophers, use them to build a powerful search engine and, after that, even a generator of new quotes!\n\nThe notebook exemplifies some of the standard usage patterns of vector search -- while showing how easy is it to get started with the vector capabilities of [Cassandra](https://cassandra.apache.org/doc/trunk/cassandra/vector-search/overview.html) / [Astra DB through CQL](https://docs.datastax.com/en/astra-serverless/docs/vector-search/quickstart.html).\n\nFor a background on using vector search and text embeddings to build a question-answering system, please check out this excellent hands-on notebook: [Question answering using embeddings](https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb).\n\n#### _Choose-your-framework_\n\nPlease note that this notebook uses the [Cassandra drivers](https://docs.datastax.com/en/developer/python-driver/latest/) and runs CQL (Cassandra Query Language) statements directly, but we cover other choices of technology to accomplish the same task. Check out this folder's [README](https://github.com/openai/openai-cookbook/tree/main/examples/vector_databases/cassandra_astradb) for other options. This notebook can run either as a Colab notebook or as a regular Jupyter notebook.\n\nTable of contents:\n- Setup\n- Get DB connection\n- Connect to OpenAI\n- Load quotes into the Vector Store\n- Use case 1: **quote search engine**\n- Use case 2: **quote generator**\n- (Optional) exploit partitioning in the Vector Store\n\n### How it works\n\n**Indexing**\n\nEach quote is made into an embedding vector with OpenAI's `Embedding`. These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing_cql](https://user-images.githubusercontent.com/14221764/282437237-1e763166-a863-4332-99b8-323ba23d1b87.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search_cql](https://user-images.githubusercontent.com/14221764/282437291-85335612-a845-444e-bed7-e4cf014a9f17.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: d8ae302a-0e61-4c13-a635-2ceb44c39f2c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d8ae302a-0e61-4c13-a635-2ceb44c39f2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6f5e4f4a-911d-48d0-916d-8133c329c1e3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f3741748ae6e0e8d448e5b5b9da843d342127003da9af3adb804c6c0311eee04\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"39239505-59da-4b08-8c1a-f5083de9717c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4ba7476368c73702a8b1893c7f1e17ce54d968d96282289ce6bc8d853e770d85\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1762, \"end_char_idx\": 4721, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "This notebook can run either as a Colab notebook or as a regular Jupyter notebook.\n\nTable of contents:\n- Setup\n- Get DB connection\n- Connect to OpenAI\n- Load quotes into the Vector Store\n- Use case 1: **quote search engine**\n- Use case 2: **quote generator**\n- (Optional) exploit partitioning in the Vector Store\n\n### How it works\n\n**Indexing**\n\nEach quote is made into an embedding vector with OpenAI's `Embedding`. These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing_cql](https://user-images.githubusercontent.com/14221764/282437237-1e763166-a863-4332-99b8-323ba23d1b87.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search_cql](https://user-images.githubusercontent.com/14221764/282437291-85335612-a845-444e-bed7-e4cf014a9f17.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282437321-881bd273-3443-4987-9a11-350d3288dd8e.png)\n\n## Setup\n\nInstall and import the necessary dependencies:\n\n\n```python\n!pip install --quiet \"cassandra-driver>=0.28.0\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nimport os\nfrom uuid import uuid4\nfrom getpass import getpass\nfrom collections import Counter\n\nfrom cassandra.cluster import Cluster\nfrom cassandra.auth import PlainTextAuthProvider\n\nimport openai\nfrom datasets import load_dataset\n```\n\n_Don't mind the next cell too much, we need it to detect Colabs and let you upload the SCB file (see below):_"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 39239505-59da-4b08-8c1a-f5083de9717c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"39239505-59da-4b08-8c1a-f5083de9717c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d8ae302a-0e61-4c13-a635-2ceb44c39f2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"6539cb1f1117e70225127a46c9530f5673fd0c90d9b80005da7c26a1610b17bd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3d6017a6-2f3e-4749-9bf5-b61f3bcb4ee1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b6ba24bff7420aa87fdfb8c7f0d1ed3b2dff57ba0744df5be5efe9487d9fdb99\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3205, \"end_char_idx\": 6192, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "_This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282437321-881bd273-3443-4987-9a11-350d3288dd8e.png)\n\n## Setup\n\nInstall and import the necessary dependencies:\n\n\n```python\n!pip install --quiet \"cassandra-driver>=0.28.0\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nimport os\nfrom uuid import uuid4\nfrom getpass import getpass\nfrom collections import Counter\n\nfrom cassandra.cluster import Cluster\nfrom cassandra.auth import PlainTextAuthProvider\n\nimport openai\nfrom datasets import load_dataset\n```\n\n_Don't mind the next cell too much, we need it to detect Colabs and let you upload the SCB file (see below):_\n\n\n```python\ntry:\n    from google.colab import files\n    IS_COLAB = True\nexcept ModuleNotFoundError:\n    IS_COLAB = False\n```\n\n## Get DB connection\n\nA couple of secrets are required to create a `Session` object (a connection to your Astra DB instance).\n\n_(Note: some steps will be slightly different on Google Colab and on local Jupyter, that's why the notebook will detect the runtime type.)_\n\n\n```python\n# Your database's Secure Connect Bundle zip file is needed:\nif IS_COLAB:\n    print('Please upload your Secure Connect Bundle zipfile: ')\n    uploaded = files.upload()\n    if uploaded:\n        astraBundleFileTitle = list(uploaded.keys())[0]\n        ASTRA_DB_SECURE_BUNDLE_PATH = os.path.join(os.getcwd(), astraBundleFileTitle)\n    else:\n        raise ValueError(\n            'Cannot proceed without Secure Connect Bundle. Please re-run the cell.'\n        )\nelse:\n    # you are running a local-jupyter notebook:\n    ASTRA_DB_SECURE_BUNDLE_PATH = input(\"Please provide the full path to your Secure Connect Bundle zipfile: \")\n\nASTRA_DB_APPLICATION_TOKEN = getpass(\"Please provide your Database Token ('AstraCS:...' string): \")\nASTRA_DB_KEYSPACE = input(\"Please provide the Keyspace name for your Database: \")\n```\n\n    Please provide the full path to your Secure Connect Bundle zipfile:  /path/to/secure-connect-DatabaseName.zip\n    Please provide your Database Token ('AstraCS:...' string):  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n    Please provide the Keyspace name for your Database:  my_keyspace"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 3d6017a6-2f3e-4749-9bf5-b61f3bcb4ee1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3d6017a6-2f3e-4749-9bf5-b61f3bcb4ee1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"39239505-59da-4b08-8c1a-f5083de9717c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"322ad0e0a9abe36c8eaac551c4453b3a10480c50fa5e69c04d1eb11c02078425\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d8005cd3-414d-430f-a3cd-7f614c4de1c1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"846fbd7358330c2354f0aed084f8166087fd00dff881eee1b6423ef6cebcc020\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4724, \"end_char_idx\": 7917, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\ntry:\n    from google.colab import files\n    IS_COLAB = True\nexcept ModuleNotFoundError:\n    IS_COLAB = False\n```\n\n## Get DB connection\n\nA couple of secrets are required to create a `Session` object (a connection to your Astra DB instance).\n\n_(Note: some steps will be slightly different on Google Colab and on local Jupyter, that's why the notebook will detect the runtime type.)_\n\n\n```python\n# Your database's Secure Connect Bundle zip file is needed:\nif IS_COLAB:\n    print('Please upload your Secure Connect Bundle zipfile: ')\n    uploaded = files.upload()\n    if uploaded:\n        astraBundleFileTitle = list(uploaded.keys())[0]\n        ASTRA_DB_SECURE_BUNDLE_PATH = os.path.join(os.getcwd(), astraBundleFileTitle)\n    else:\n        raise ValueError(\n            'Cannot proceed without Secure Connect Bundle. Please re-run the cell.'\n        )\nelse:\n    # you are running a local-jupyter notebook:\n    ASTRA_DB_SECURE_BUNDLE_PATH = input(\"Please provide the full path to your Secure Connect Bundle zipfile: \")\n\nASTRA_DB_APPLICATION_TOKEN = getpass(\"Please provide your Database Token ('AstraCS:...' string): \")\nASTRA_DB_KEYSPACE = input(\"Please provide the Keyspace name for your Database: \")\n```\n\n    Please provide the full path to your Secure Connect Bundle zipfile:  /path/to/secure-connect-DatabaseName.zip\n    Please provide your Database Token ('AstraCS:...' string):  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n    Please provide the Keyspace name for your Database:  my_keyspace\n\n\n### Creation of the DB connection\n\nThis is how you create a connection to Astra DB:\n\n_(Incidentally, you could also use any Cassandra cluster (as long as it provides Vector capabilities), just by [changing the parameters](https://docs.datastax.com/en/developer/python-driver/latest/getting_started/#connecting-to-cassandra) to the following `Cluster` instantiation.)_\n\n\n```python\n# Don't mind the \"Closing connection\" error after \"downgrading protocol...\" messages you may see,\n# it is really just a warning: the connection will work smoothly.\ncluster = Cluster(\n    cloud={\n        \"secure_connect_bundle\": ASTRA_DB_SECURE_BUNDLE_PATH,\n    },\n    auth_provider=PlainTextAuthProvider(\n        \"token\",\n        ASTRA_DB_APPLICATION_TOKEN,\n    ),\n)\n\nsession = cluster.connect()\nkeyspace = ASTRA_DB_KEYSPACE\n```\n\n### Creation of the Vector table in CQL\n\nYou need a table which support vectors and is equipped with metadata. Call it \"philosophers_cql\".\n\nEach row will store: a quote, its vector embedding, the quote author and a set of \"tags\". You also need a primary key to ensure uniqueness of rows.\n\nThe following is the full CQL command that creates the table (check out [this page](https://docs.datastax.com/en/dse/6.7/cql/cql/cqlQuickReference.html) for more on the CQL syntax of this and the following statements):\n\n\n```python\ncreate_table_statement = f\"\"\"CREATE TABLE IF NOT EXISTS {keyspace}.philosophers_cql (\n    quote_id UUID PRIMARY KEY,\n    body TEXT,\n    embedding_vector VECTOR<FLOAT, 1536>,\n    author TEXT,\n    tags SET<TEXT>\n);\"\"\"\n```\n\nPass this statement to your database Session to execute it:\n\n\n```python\nsession.execute(create_table_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7feee37b3460>"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: d8005cd3-414d-430f-a3cd-7f614c4de1c1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d8005cd3-414d-430f-a3cd-7f614c4de1c1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3d6017a6-2f3e-4749-9bf5-b61f3bcb4ee1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e47db5e16ea8b2a4de492c4092208819fd932baf01b98fceb0335fc3b0bd938d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"35eec181-ca45-4b8c-aee2-e51620e198dc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"98bd0cbdc96ccb5b62bf6e326b1a22570532a217d06593394b9141832f311119\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6565, \"end_char_idx\": 9807, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\n# Don't mind the \"Closing connection\" error after \"downgrading protocol...\" messages you may see,\n# it is really just a warning: the connection will work smoothly.\ncluster = Cluster(\n    cloud={\n        \"secure_connect_bundle\": ASTRA_DB_SECURE_BUNDLE_PATH,\n    },\n    auth_provider=PlainTextAuthProvider(\n        \"token\",\n        ASTRA_DB_APPLICATION_TOKEN,\n    ),\n)\n\nsession = cluster.connect()\nkeyspace = ASTRA_DB_KEYSPACE\n```\n\n### Creation of the Vector table in CQL\n\nYou need a table which support vectors and is equipped with metadata. Call it \"philosophers_cql\".\n\nEach row will store: a quote, its vector embedding, the quote author and a set of \"tags\". You also need a primary key to ensure uniqueness of rows.\n\nThe following is the full CQL command that creates the table (check out [this page](https://docs.datastax.com/en/dse/6.7/cql/cql/cqlQuickReference.html) for more on the CQL syntax of this and the following statements):\n\n\n```python\ncreate_table_statement = f\"\"\"CREATE TABLE IF NOT EXISTS {keyspace}.philosophers_cql (\n    quote_id UUID PRIMARY KEY,\n    body TEXT,\n    embedding_vector VECTOR<FLOAT, 1536>,\n    author TEXT,\n    tags SET<TEXT>\n);\"\"\"\n```\n\nPass this statement to your database Session to execute it:\n\n\n```python\nsession.execute(create_table_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7feee37b3460>\n\n\n\n#### Add a vector index for ANN search\n\nIn order to run ANN (approximate-nearest-neighbor) searches on the vectors in the table, you need to create a specific index on the `embedding_vector` column.\n\n_When creating the index, you can [optionally choose](https://docs.datastax.com/en/astra-serverless/docs/vector-search/cql.html#_create_the_vector_schema_and_load_the_data_into_the_database) the \"similarity function\" used to compute vector distances: since for unit-length vectors (such as those from OpenAI) the \"cosine difference\" is the same as the \"dot product\", you'll use the latter which is computationally less expensive._\n\nRun this CQL statement:\n\n\n```python\ncreate_vector_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_embedding_vector\n    ON {keyspace}.philosophers_cql (embedding_vector)\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'\n    WITH OPTIONS = {{'similarity_function' : 'dot_product'}};\n\"\"\"\n# Note: the double '{{' and '}}' are just the F-string escape sequence for '{' and '}'\n\nsession.execute(create_vector_index_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7feeefd3da00>\n\n\n\n#### Add indexes for author and tag filtering\n\nThat is enough to run vector searches on the table ... but you want to be able to optionally specify an author and/or some tags to restrict the quote search. Create two other indexes to support this:\n\n\n```python\ncreate_author_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_author\n    ON {keyspace}.philosophers_cql (author)\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';\n\"\"\"\nsession.execute(create_author_index_statement)\n\ncreate_tags_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_tags\n    ON {keyspace}.philosophers_cql (VALUES(tags))\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';\n\"\"\"\nsession.execute(create_tags_index_statement)\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 35eec181-ca45-4b8c-aee2-e51620e198dc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"35eec181-ca45-4b8c-aee2-e51620e198dc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d8005cd3-414d-430f-a3cd-7f614c4de1c1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9104e5a7515dae27406e8134c22657923cb42a5c92ff24b37c148bb3b2fe3e83\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d8cfee42-5879-4332-b8ee-695221253a43\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8a90b60290cc805efdc757e3284a312f389a183e886aa73742ee3e95b4fbff95\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8579, \"end_char_idx\": 11519, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\ncreate_vector_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_embedding_vector\n    ON {keyspace}.philosophers_cql (embedding_vector)\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'\n    WITH OPTIONS = {{'similarity_function' : 'dot_product'}};\n\"\"\"\n# Note: the double '{{' and '}}' are just the F-string escape sequence for '{' and '}'\n\nsession.execute(create_vector_index_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7feeefd3da00>\n\n\n\n#### Add indexes for author and tag filtering\n\nThat is enough to run vector searches on the table ... but you want to be able to optionally specify an author and/or some tags to restrict the quote search. Create two other indexes to support this:\n\n\n```python\ncreate_author_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_author\n    ON {keyspace}.philosophers_cql (author)\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';\n\"\"\"\nsession.execute(create_author_index_statement)\n\ncreate_tags_index_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_tags\n    ON {keyspace}.philosophers_cql (VALUES(tags))\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';\n\"\"\"\nsession.execute(create_tags_index_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7fef2c64af70>\n\n\n\n## Connect to OpenAI\n\n### Set up your secret key\n\n\n```python\nOPENAI_API_KEY = getpass(\"Please enter your OpenAI API Key: \")\n```\n\n    Please enter your OpenAI API Key:  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nprint(f\"len(result.data)              = {len(result.data)}\")\nprint(f\"result.data[1].embedding      = {str(result.data[1].embedding)[:55]}...\")\nprint(f\"len(result.data[1].embedding) = {len(result.data[1].embedding)}\")\n```\n\n    len(result.data)              = 2\n    result.data[1].embedding      = [-0.0108176339417696, 0.0013546717818826437, 0.00362232...\n    len(result.data[1].embedding) = 1536\n\n\n## Load quotes into the Vector Store\n\nGet a dataset with the quotes. _(We adapted and augmented the data from [this Kaggle dataset](https://www.kaggle.com/datasets/mertbozkurt5/quotes-by-philosophers), ready to use in this demo.)_\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: d8cfee42-5879-4332-b8ee-695221253a43
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d8cfee42-5879-4332-b8ee-695221253a43\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"35eec181-ca45-4b8c-aee2-e51620e198dc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"44554f3c3934b9a96349b85bbd4bffea307a188b8502cab6e422fe42dd60716d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e5b200b1-c2ba-429e-8fa5-61b12fbbc383\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4f2d58adf9afe963b75e4e36f0e53cc4fa77ca26b1881004bd6cde402b7dfce6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10046, \"end_char_idx\": 12814, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nprint(f\"len(result.data)              = {len(result.data)}\")\nprint(f\"result.data[1].embedding      = {str(result.data[1].embedding)[:55]}...\")\nprint(f\"len(result.data[1].embedding) = {len(result.data[1].embedding)}\")\n```\n\n    len(result.data)              = 2\n    result.data[1].embedding      = [-0.0108176339417696, 0.0013546717818826437, 0.00362232...\n    len(result.data[1].embedding) = 1536\n\n\n## Load quotes into the Vector Store\n\nGet a dataset with the quotes. _(We adapted and augmented the data from [this Kaggle dataset](https://www.kaggle.com/datasets/mertbozkurt5/quotes-by-philosophers), ready to use in this demo.)_\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Insert quotes into vector store\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata planned for later use.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\nThe DB write is accomplished with a CQL statement. But since you'll run this particular insertion several times (albeit with different values), it's best to _prepare_ the statement and then just run it over and over.\n\n_(Note: for faster insertion, the Cassandra drivers would let you do concurrent inserts, which we don't do here for a more straightforward demo code.)_"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: e5b200b1-c2ba-429e-8fa5-61b12fbbc383
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e5b200b1-c2ba-429e-8fa5-61b12fbbc383\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d8cfee42-5879-4332-b8ee-695221253a43\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9b3362b4817093b41310615d896f72b137f795bf4f3051de7ab06ef2ff67d2c4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"48868d49-2ae8-4d27-9d17-589964e86716\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0cf4ba4945eeee8734ce4368d0353141f2dcae9f3243b5adb8f5cb617b2b5fd8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11189, \"end_char_idx\": 15199, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Insert quotes into vector store\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata planned for later use.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\nThe DB write is accomplished with a CQL statement. But since you'll run this particular insertion several times (albeit with different values), it's best to _prepare_ the statement and then just run it over and over.\n\n_(Note: for faster insertion, the Cassandra drivers would let you do concurrent inserts, which we don't do here for a more straightforward demo code.)_\n\n\n```python\nprepared_insertion = session.prepare(\n    f\"INSERT INTO {keyspace}.philosophers_cql (quote_id, author, body, embedding_vector, tags) VALUES (?, ?, ?, ?, ?);\"\n)\n\nBATCH_SIZE = 20\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries:\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the rows for insertion\n    print(\"B \", end=\"\")\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = set()\n        author = authors_list[entry_idx]\n        quote = quotes_list[entry_idx]\n        quote_id = uuid4()  # a new random ID for each quote. In a production app you'll want to have better control...\n        session.execute(\n            prepared_insertion,\n            (quote_id, author, quote, emb_result.embedding, tags),\n        )\n        print(\"*\", end=\"\")\n    print(f\" done ({len(b_emb_results.data)})\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries:\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ******************** done (20)\n    B ********** done (10)\n    \n    Finished storing entries."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 48868d49-2ae8-4d27-9d17-589964e86716
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"48868d49-2ae8-4d27-9d17-589964e86716\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e5b200b1-c2ba-429e-8fa5-61b12fbbc383\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ca9db875183d25189a8e44a4948849a8320da8028929437ab618a36f53799614\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"513e1320-c3c3-48ca-9b59-9247355ffefc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a010ca8a2abc0c035cd080ff5b4b38de855471586df5854e3b0f5c3c9af6e1dd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15202, \"end_char_idx\": 18357, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "## Use case 1: **quote search engine**\n\nFor the quote-search functionality, you need first to make the input quote into a vector, and then use it to query the store (besides handling the optional metadata into the search call, that is).\n\nEncapsulate the search-engine functionality into a function for ease of re-use:\n\n\n```python\ndef find_quote_and_author(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    # depending on what conditions are passed, the WHERE clause in the statement may vary.\n    where_clauses = []\n    where_values = []\n    if author:\n        where_clauses += [\"author = %s\"]\n        where_values += [author]\n    if tags:\n        for tag in tags:\n            where_clauses += [\"tags CONTAINS %s\"]\n            where_values += [tag]\n    # The reason for these two lists above is that when running the CQL search statement the values passed\n    # must match the sequence of \"?\" marks in the statement.\n    if where_clauses:\n        search_statement = f\"\"\"SELECT body, author FROM {keyspace}.philosophers_cql\n            WHERE {' AND '.join(where_clauses)}\n            ORDER BY embedding_vector ANN OF %s\n            LIMIT %s;\n        \"\"\"\n    else:\n        search_statement = f\"\"\"SELECT body, author FROM {keyspace}.philosophers_cql\n            ORDER BY embedding_vector ANN OF %s\n            LIMIT %s;\n        \"\"\"\n    # For best performance, one should keep a cache of prepared statements (see the insertion code above)\n    # for the various possible statements used here.\n    # (We'll leave it as an exercise to the reader to avoid making this code too long.\n    # Remember: to prepare a statement you use '?' instead of '%s'.)\n    query_values = tuple(where_values + [query_vector] + [n])\n    result_rows = session.execute(search_statement, query_values)\n    return [\n        (result_row.body, result_row.author)\n        for result_row in result_rows\n    ]\n```\n\n### Putting search to test\n\nPassing just a quote:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nSearch constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 513e1320-c3c3-48ca-9b59-9247355ffefc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"513e1320-c3c3-48ca-9b59-9247355ffefc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"48868d49-2ae8-4d27-9d17-589964e86716\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a570e2a6e6c1858f6fe1bf9b4ed2dc6c2ce33ad8f6e975ae846150f3b6786387\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b1dcc445-502a-423f-8e03-341cddcc83cc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cd4547429557361b73151049a4912257ae0f1bf1b53bd91711af637f6a52a8bf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17262, \"end_char_idx\": 19920, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nSearch constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('Mankind will never see an end of trouble until lovers of wisdom come to hold political power, or the holders of power become lovers of wisdom',\n      'plato'),\n     ('Everything the State says is a lie, and everything it has it has stolen.',\n      'nietzsche')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"similarity\" between the query and each result, and then set a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results:\n\n_Note (for the mathematically inclined): this value is **a rescaling between zero and one** of the cosine difference between the vectors, i.e. of the scalar product divided by the product of the norms of the two vectors. In other words, this is 0 for opposite-facing vecors and +1 for parallel vectors. For other measures of similarity, check the [documentation](https://docs.datastax.com/en/astra-serverless/docs/vector-search/cql.html#_create_the_vector_schema_and_load_the_data_into_the_database) -- and keep in mind that the metric in the `SELECT` query should match the one used when creating the index earlier for meaningful, ordered results._"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: b1dcc445-502a-423f-8e03-341cddcc83cc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"b1dcc445-502a-423f-8e03-341cddcc83cc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"513e1320-c3c3-48ca-9b59-9247355ffefc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3a52e5f840d6ad8c22f822e183160be223244d104dbc0e9b4c6f545fcc21bf11\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"82904b95-2724-4580-8dab-b5e7ce8bff7b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"35e0a28c622327eb5917eb9ae789bfd488cd3dbba9af0f5ba0158af98001d469\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18184, \"end_char_idx\": 21249, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "Search constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('Mankind will never see an end of trouble until lovers of wisdom come to hold political power, or the holders of power become lovers of wisdom',\n      'plato'),\n     ('Everything the State says is a lie, and everything it has it has stolen.',\n      'nietzsche')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"similarity\" between the query and each result, and then set a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results:\n\n_Note (for the mathematically inclined): this value is **a rescaling between zero and one** of the cosine difference between the vectors, i.e. of the scalar product divided by the product of the norms of the two vectors. In other words, this is 0 for opposite-facing vecors and +1 for parallel vectors. For other measures of similarity, check the [documentation](https://docs.datastax.com/en/astra-serverless/docs/vector-search/cql.html#_create_the_vector_schema_and_load_the_data_into_the_database) -- and keep in mind that the metric in the `SELECT` query should match the one used when creating the index earlier for meaningful, ordered results._\n\n\n```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nsimilarity_threshold = 0.92\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\n# Once more: remember to prepare your statements in production for greater performance...\n\nsearch_statement = f\"\"\"SELECT body, similarity_dot_product(embedding_vector, %s) as similarity\n    FROM {keyspace}.philosophers_cql\n    ORDER BY embedding_vector ANN OF %s\n    LIMIT %s;\n\"\"\"\nquery_values = (quote_vector, quote_vector, 8)\n\nresult_rows = session.execute(search_statement, query_values)\nresults = [\n    (result_row.body, result_row.similarity)\n    for result_row in result_rows\n    if result_row.similarity >= similarity_threshold\n]\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, (r_body, r_similarity) in enumerate(results):\n    print(f\"    {idx}. [similarity={r_similarity:.3f}] \\\"{r_body[:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [similarity=0.927] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [similarity=0.922] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [similarity=0.920] \"At his best, man is the noblest of all animals; separated from law and...\""}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 82904b95-2724-4580-8dab-b5e7ce8bff7b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"82904b95-2724-4580-8dab-b5e7ce8bff7b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b1dcc445-502a-423f-8e03-341cddcc83cc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3e18780cd7099f1b9e667062efb27a34c06f21fdb8c1659b61cae3cc41737fb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"889ef492-0629-4041-bc8a-3787b03db749\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8f8e89e495ea0bf57ec41a8d98b454633d0e930ca2ddb83ddb178bd5346129ea\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19923, \"end_char_idx\": 21974, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nsimilarity_threshold = 0.92\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\n# Once more: remember to prepare your statements in production for greater performance...\n\nsearch_statement = f\"\"\"SELECT body, similarity_dot_product(embedding_vector, %s) as similarity\n    FROM {keyspace}.philosophers_cql\n    ORDER BY embedding_vector ANN OF %s\n    LIMIT %s;\n\"\"\"\nquery_values = (quote_vector, quote_vector, 8)\n\nresult_rows = session.execute(search_statement, query_values)\nresults = [\n    (result_row.body, result_row.similarity)\n    for result_row in result_rows\n    if result_row.similarity >= similarity_threshold\n]\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, (r_body, r_similarity) in enumerate(results):\n    print(f\"    {idx}. [similarity={r_similarity:.3f}] \\\"{r_body[:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [similarity=0.927] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [similarity=0.922] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [similarity=0.920] \"At his best, man is the noblest of all animals; separated from law and...\"\n\n\n## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task.\n\n\n```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 889ef492-0629-4041-bc8a-3787b03db749
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"889ef492-0629-4041-bc8a-3787b03db749\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"82904b95-2724-4580-8dab-b5e7ce8bff7b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"031b28ae1b763aa335348dc1c0db1b5221754ba63f64547ffdfcbfdd4177c921\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"409f9e55-4a3f-4d0a-b885-706ecec1fea4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"18b6c30867e69f224bf4bb2bb5a7d6aed099f4a0e97c5b7ac60a2a2edb176206\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21252, \"end_char_idx\": 23809, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task.\n\n\n```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):\n\n\n```python\ndef generate_quote(topic, n=2, author=None, tags=None):\n    quotes = find_quote_and_author(query_quote=topic, n=n, author=author, tags=tags)\n    if quotes:\n        prompt = generation_prompt_template.format(\n            topic=topic,\n            examples=\"\\n\".join(f\"  - {quote[0]}\" for quote in quotes),\n        )\n        # a little logging:\n        print(\"** quotes found:\")\n        for q, a in quotes:\n            print(f\"**    - {q} ({a})\")\n        print(\"** end of logging\")\n        #\n        response = client.chat.completions.create(\n            model=completion_model_name,\n            messages=[{\"role\": \"user\", \"content\": prompt}],\n            temperature=0.7,\n            max_tokens=320,\n        )\n        return response.choices[0].message.content.replace('\"', '').strip()\n    else:\n        print(\"** no quotes found.\")\n        return None\n```\n\n_Note: similar to the case of the embedding computation, the code for the Chat Completion API would be slightly different for OpenAI prior to v1.0._\n\n#### Putting quote generation to test\n\nJust passing a text (a \"quote\", but one can actually just suggest a topic since its vector embedding will still end up at the right place in the vector space):\n\n\n```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    True politics is not the pursuit of power, but the cultivation of virtue for the betterment of all.\n\n\nUse inspiration from just a single philosopher:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 409f9e55-4a3f-4d0a-b885-706ecec1fea4
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"409f9e55-4a3f-4d0a-b885-706ecec1fea4\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"889ef492-0629-4041-bc8a-3787b03db749\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c6f98c5b8fd9de5a22d4f58588aa5147984fde473302c48f508326b3d7d1bc55\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7b504bae-1a21-4eb6-a15e-3c51b3ef0197\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"559834002524a2623061746df5b16836c5c72f794672d9f40dd6840bbec767fe\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23194, \"end_char_idx\": 26107, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    True politics is not the pursuit of power, but the cultivation of virtue for the betterment of all.\n\n\nUse inspiration from just a single philosopher:\n\n\n```python\nq_topic = generate_quote(\"animals\", author=\"schopenhauer\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Because Christian morality leaves animals out of account, they are at once outlawed in philosophical morals; they are mere 'things,' mere means to any ends whatsoever. They can therefore be used for vivisection, hunting, coursing, bullfights, and horse racing, and can be whipped to death as they struggle along with heavy carts of stone. Shame on such a morality that is worthy of pariahs, and that fails to recognize the eternal essence that exists in every living thing, and shines forth with inscrutable significance from all eyes that see the sun! (schopenhauer)\n    **    - The assumption that animals are without rights, and the illusion that our treatment of them has no moral significance, is a positively outrageous example of Western crudity and barbarity. Universal compassion is the only guarantee of morality. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    Do not judge the worth of a soul by its outward form, for within every animal lies an eternal essence that deserves our compassion and respect.\n\n\n## (Optional) **Partitioning**\n\nThere's an interesting topic to examine before completing this quickstart. While, generally, tags and quotes can be in any relationship (e.g. a quote having multiple tags), _authors_ are effectively an exact grouping (they define a \"disjoint partitioning\" on the set of quotes): each quote has exactly one author (for us, at least).\n\nNow, suppose you know in advance your application will usually (or always) run queries on a _single author_. Then you can take full advantage of the underlying database structure: if you group quotes in **partitions** (one per author), vector queries on just an author will use less resources and return much faster.\n\nWe'll not dive into the details here, which have to do with the Cassandra storage internals: the important message is that **if your queries are run within a group, consider partitioning accordingly to boost performance**.\n\nYou'll now see this choice in action.\n\nThe partitioning per author calls for a new table schema: create a new table called \"philosophers_cql_partitioned\", along with the necessary indexes:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 7b504bae-1a21-4eb6-a15e-3c51b3ef0197
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7b504bae-1a21-4eb6-a15e-3c51b3ef0197\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"409f9e55-4a3f-4d0a-b885-706ecec1fea4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b26798290da598cd146d242f0641b1a8ab89e88cd749304f877844cfb7485462\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fe99ae43-05ba-4df5-ac63-e025bade9fab\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e684e0bc6b577dd11b0f0ac541e6af48426e7bf8551ac19a56a67e833b2b5c3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 25011, \"end_char_idx\": 28012, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "## (Optional) **Partitioning**\n\nThere's an interesting topic to examine before completing this quickstart. While, generally, tags and quotes can be in any relationship (e.g. a quote having multiple tags), _authors_ are effectively an exact grouping (they define a \"disjoint partitioning\" on the set of quotes): each quote has exactly one author (for us, at least).\n\nNow, suppose you know in advance your application will usually (or always) run queries on a _single author_. Then you can take full advantage of the underlying database structure: if you group quotes in **partitions** (one per author), vector queries on just an author will use less resources and return much faster.\n\nWe'll not dive into the details here, which have to do with the Cassandra storage internals: the important message is that **if your queries are run within a group, consider partitioning accordingly to boost performance**.\n\nYou'll now see this choice in action.\n\nThe partitioning per author calls for a new table schema: create a new table called \"philosophers_cql_partitioned\", along with the necessary indexes:\n\n\n```python\ncreate_table_p_statement = f\"\"\"CREATE TABLE IF NOT EXISTS {keyspace}.philosophers_cql_partitioned (\n    author TEXT,\n    quote_id UUID,\n    body TEXT,\n    embedding_vector VECTOR<FLOAT, 1536>,\n    tags SET<TEXT>,\n    PRIMARY KEY ( (author), quote_id )\n) WITH CLUSTERING ORDER BY (quote_id ASC);\"\"\"\n\nsession.execute(create_table_p_statement)\n\ncreate_vector_index_p_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_embedding_vector_p\n    ON {keyspace}.philosophers_cql_partitioned (embedding_vector)\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'\n    WITH OPTIONS = {{'similarity_function' : 'dot_product'}};\n\"\"\"\n\nsession.execute(create_vector_index_p_statement)\n\ncreate_tags_index_p_statement = f\"\"\"CREATE CUSTOM INDEX IF NOT EXISTS idx_tags_p\n    ON {keyspace}.philosophers_cql_partitioned (VALUES(tags))\n    USING 'org.apache.cassandra.index.sai.StorageAttachedIndex';\n\"\"\"\nsession.execute(create_tags_index_p_statement)\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7fef149d7940>\n\n\n\nNow repeat the compute-embeddings-and-insert step on the new table.\n\nYou could use the very same insertion code as you did earlier, because the differences are hidden \"behind the scenes\": the database will store the inserted rows differently according to the partitioning scheme of this new table.\n\nHowever, by way of demonstration, you will take advantage of a handy facility offered by the Cassandra drivers to easily run several queries (in this case, `INSERT`s) concurrently. This is something that Cassandra / Astra DB through CQL supports very well and can lead to a significant speedup, with very little changes in the client code.\n\n_(Note: one could additionally have cached the embeddings computed previously to save a few API tokens -- here, however, we wanted to keep the code easier to inspect.)_\n\n\n```python\nfrom cassandra.concurrent import execute_concurrent_with_args\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: fe99ae43-05ba-4df5-ac63-e025bade9fab
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"fe99ae43-05ba-4df5-ac63-e025bade9fab\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7b504bae-1a21-4eb6-a15e-3c51b3ef0197\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e0253ee5d1a3e4df8635c125d936dd31c90a715d8ee6ce12c88113d630c8b3da\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"63efe815-c295-4af0-bcea-9ca69954e52f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c8a725cd29aabffd2a427b3396cf76958d07bdc84351a4a5a7f68749950cd885\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 27075, \"end_char_idx\": 30229, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "<cassandra.cluster.ResultSet at 0x7fef149d7940>\n\n\n\nNow repeat the compute-embeddings-and-insert step on the new table.\n\nYou could use the very same insertion code as you did earlier, because the differences are hidden \"behind the scenes\": the database will store the inserted rows differently according to the partitioning scheme of this new table.\n\nHowever, by way of demonstration, you will take advantage of a handy facility offered by the Cassandra drivers to easily run several queries (in this case, `INSERT`s) concurrently. This is something that Cassandra / Astra DB through CQL supports very well and can lead to a significant speedup, with very little changes in the client code.\n\n_(Note: one could additionally have cached the embeddings computed previously to save a few API tokens -- here, however, we wanted to keep the code easier to inspect.)_\n\n\n```python\nfrom cassandra.concurrent import execute_concurrent_with_args\n```\n\n\n```python\nprepared_insertion = session.prepare(\n    f\"INSERT INTO {keyspace}.philosophers_cql_partitioned (quote_id, author, body, embedding_vector, tags) VALUES (?, ?, ?, ?, ?);\"\n)\n\nBATCH_SIZE = 50\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries:\")\nfor batch_i in range(num_batches):\n    print(\"[...\", end=\"\")\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare this batch's entries for insertion\n    tuples_to_insert = []\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = set()\n        author = authors_list[entry_idx]\n        quote = quotes_list[entry_idx]\n        quote_id = uuid4()  # a new random ID for each quote. In a production app you'll want to have better control...\n        # append a *tuple* to the list, and in the tuple the values are ordered to match \"?\" in the prepared statement:\n        tuples_to_insert.append((quote_id, author, quote, emb_result.embedding, tags))\n    # insert the batch at once through the driver's concurrent primitive\n    conc_results = execute_concurrent_with_args(\n        session,\n        prepared_insertion,\n        tuples_to_insert,\n    )\n    # check that all insertions succeed (better to always do this):\n    if any([not success for success, _ in conc_results]):\n        print(\"Something failed during the insertions!\")\n    else:\n        print(f\"{len(b_emb_results.data)}] \", end=\"\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries:\n    [...50] [...50] [...50] [...50] [...50] [...50] [...50] [...50] [...50] \n    Finished storing entries.\n\n\nDespite the different table schema, the DB query behind the similarity search is essentially the same:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 63efe815-c295-4af0-bcea-9ca69954e52f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"63efe815-c295-4af0-bcea-9ca69954e52f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fe99ae43-05ba-4df5-ac63-e025bade9fab\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"defa52d24270babdd12166482baee4950b4cb1ac45bec0e608321be16e3acaed\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0018ce3e-eb6b-4f9d-8149-60c66e4b06c5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7250c5ac93bd71b4c3e2a9c5bdc4e077fa886918fa3c3ad1c4cf64718e8bb1a3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 30127, \"end_char_idx\": 32570, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "Despite the different table schema, the DB query behind the similarity search is essentially the same:\n\n\n```python\ndef find_quote_and_author_p(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    # Depending on what conditions are passed, the WHERE clause in the statement may vary.\n    # Construct it accordingly:\n    where_clauses = []\n    where_values = []\n    if author:\n        where_clauses += [\"author = %s\"]\n        where_values += [author]\n    if tags:\n        for tag in tags:\n            where_clauses += [\"tags CONTAINS %s\"]\n            where_values += [tag]\n    if where_clauses:\n        search_statement = f\"\"\"SELECT body, author FROM {keyspace}.philosophers_cql_partitioned\n            WHERE {' AND '.join(where_clauses)}\n            ORDER BY embedding_vector ANN OF %s\n            LIMIT %s;\n        \"\"\"\n    else:\n        search_statement = f\"\"\"SELECT body, author FROM {keyspace}.philosophers_cql_partitioned\n            ORDER BY embedding_vector ANN OF %s\n            LIMIT %s;\n        \"\"\"\n    query_values = tuple(where_values + [query_vector] + [n])\n    result_rows = session.execute(search_statement, query_values)\n    return [\n        (result_row.body, result_row.author)\n        for result_row in result_rows\n    ]\n```\n\nThat's it: the new table still supports the \"generic\" similarity searches all right ...\n\n\n```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\n... but it's when an author is specified that you would notice a _huge_ performance advantage:\n\n\n```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 0018ce3e-eb6b-4f9d-8149-60c66e4b06c5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB", "tokens": 7788, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0018ce3e-eb6b-4f9d-8149-60c66e4b06c5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"2b7083e0-a2c3-4d46-81d5-0aadf6884348\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c612319605ec9e5f26c9165ca32a7036b2f07e2ac5ee91dc11787d133a1fcb44\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"63efe815-c295-4af0-bcea-9ca69954e52f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB\", \"tokens\": 7788, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"41095687a58c8947e124ef269d26a8935eb621596e0968c0a60d77fd3c7ec6c5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 31585, \"end_char_idx\": 33995, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "ref_doc_id": "2b7083e0-a2c3-4d46-81d5-0aadf6884348", "chroma:document": "```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\n... but it's when an author is specified that you would notice a _huge_ performance advantage:\n\n\n```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nWell, you _would_ notice a performance gain, if you had a realistic-size dataset. In this demo, with a few tens of entries, there's no noticeable difference -- but you get the idea.\n\n## Conclusion\n\nCongratulations! You have learned how to use OpenAI for vector embeddings and Astra DB / Cassandra for storage in order to build a sophisticated philosophical search engine and quote generator.\n\nThis example used the [Cassandra drivers](https://docs.datastax.com/en/developer/python-driver/latest/) and runs CQL (Cassandra Query Language) statements directly to interface with the Vector Store - but this is not the only choice. Check the [README](https://github.com/openai/openai-cookbook/tree/main/examples/vector_databases/cassandra_astradb) for other options and integration with popular frameworks.\n\nTo find out more on how Astra DB's Vector Search capabilities can be a key ingredient in your ML/GenAI applications, visit [Astra DB](https://docs.datastax.com/en/astra-serverless/docs/vector-search/overview.html)'s web page on the topic.\n\n## Cleanup\n\nIf you want to remove all resources used for this demo, run this cell (_warning: this will delete the tables and the data inserted in them!_):\n\n\n```python\nsession.execute(f\"DROP TABLE IF EXISTS {keyspace}.philosophers_cql;\")\nsession.execute(f\"DROP TABLE IF EXISTS {keyspace}.philosophers_cql_partitioned;\")\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7fef149096a0>"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 6f01392e-a404-4bb3-8eda-f7ab26775281
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6f01392e-a404-4bb3-8eda-f7ab26775281\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1bc66e65-401c-4e04-913b-32c13bc7d474\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"63d89d7e1f58572f09ed18c4f10302344b73c1eb419266b310ab6fda313e9fe9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3274, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "# Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\n\n### CassIO version\n\nIn this quickstart you will learn how to build a \"philosophy quote finder & generator\" using OpenAI's vector embeddings and [Apache Cassandra\u00ae](https://cassandra.apache.org), or equivalently DataStax [Astra DB through CQL](https://docs.datastax.com/en/astra-serverless/docs/vector-search/quickstart.html), as the vector store for data persistence.\n\nThe basic workflow of this notebook is outlined below. You will evaluate and store the vector embeddings for a number of quotes by famous philosophers, use them to build a powerful search engine and, after that, even a generator of new quotes!\n\nThe notebook exemplifies some of the standard usage patterns of vector search -- while showing how easy is it to get started with the vector capabilities of [Cassandra](https://cassandra.apache.org/doc/trunk/cassandra/vector-search/overview.html) / [Astra DB through CQL](https://docs.datastax.com/en/astra-serverless/docs/vector-search/quickstart.html).\n\nFor a background on using vector search and text embeddings to build a question-answering system, please check out this excellent hands-on notebook: [Question answering using embeddings](https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb).\n\n#### _Choose-your-framework_\n\nPlease note that this notebook uses the [CassIO library](https://cassio.org), but we cover other choices of technology to accomplish the same task. Check out this folder's [README](https://github.com/openai/openai-cookbook/tree/main/examples/vector_databases/cassandra_astradb) for other options. This notebook can run either as a Colab notebook or as a regular Jupyter notebook.\n\nTable of contents:\n- Setup\n- Get DB connection\n- Connect to OpenAI\n- Load quotes into the Vector Store\n- Use case 1: **quote search engine**\n- Use case 2: **quote generator**\n- (Optional) exploit partitioning in the Vector Store\n\n### How it works\n\n**Indexing**\n\nEach quote is made into an embedding vector with OpenAI's `Embedding`. These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing](https://user-images.githubusercontent.com/14221764/282440878-dc3ed680-7d0e-4b30-9a74-d2d66a7394f7.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search](https://user-images.githubusercontent.com/14221764/282440908-683e3ee1-0bf1-46b3-8621-86c31fc7f9c9.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 1bc66e65-401c-4e04-913b-32c13bc7d474
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1bc66e65-401c-4e04-913b-32c13bc7d474\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6f01392e-a404-4bb3-8eda-f7ab26775281\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cc0bd0b46c4466a5d8ce7cbcd87d74087449cb2530f8f43348ef9e824f8d2d46\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ad8a46ab-85c1-4d11-8845-429593ee1cbf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e066639a27df3a783c88b5a99d58b44a03f292a74a25a563e24033ee2302d39d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1672, \"end_char_idx\": 4246, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "This notebook can run either as a Colab notebook or as a regular Jupyter notebook.\n\nTable of contents:\n- Setup\n- Get DB connection\n- Connect to OpenAI\n- Load quotes into the Vector Store\n- Use case 1: **quote search engine**\n- Use case 2: **quote generator**\n- (Optional) exploit partitioning in the Vector Store\n\n### How it works\n\n**Indexing**\n\nEach quote is made into an embedding vector with OpenAI's `Embedding`. These are saved in the Vector Store for later use in searching. Some metadata, including the author's name and a few other pre-computed tags, are stored alongside, to allow for search customization.\n\n![1_vector_indexing](https://user-images.githubusercontent.com/14221764/282440878-dc3ed680-7d0e-4b30-9a74-d2d66a7394f7.png)\n\n**Search**\n\nTo find a quote similar to the provided search quote, the latter is made into an embedding vector on the fly, and this vector is used to query the store for similar vectors ... i.e. similar quotes that were previously indexed. The search can optionally be constrained by additional metadata (\"find me quotes by Spinoza similar to this one ...\").\n\n![2_vector_search](https://user-images.githubusercontent.com/14221764/282440908-683e3ee1-0bf1-46b3-8621-86c31fc7f9c9.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282440927-d56f36eb-d611-4342-8026-7736edc6f5c9.png)\n\n## Setup\n\nFirst install some required packages:\n\n\n```python\n!pip install --quiet \"cassio>=0.1.3\" \"openai>=1.0.0\" datasets\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: ad8a46ab-85c1-4d11-8845-429593ee1cbf
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ad8a46ab-85c1-4d11-8845-429593ee1cbf\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1bc66e65-401c-4e04-913b-32c13bc7d474\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"19b6f2faef47131a6fa8587cd08b2a33b202779078b71c32bf7ee449669a3378\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ad7251e5-15ab-4d88-8275-a9faddda9e71\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e05764c860cfdd91fb216182a0e4081a2031f0d1899c736affb67bdc921c9af1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2773, \"end_char_idx\": 5604, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "![2_vector_search](https://user-images.githubusercontent.com/14221764/282440908-683e3ee1-0bf1-46b3-8621-86c31fc7f9c9.png)\n\nThe key point here is that \"quotes similar in content\" translates, in vector space, to vectors that are metrically close to each other: thus, vector similarity search effectively implements semantic similarity. _This is the key reason vector embeddings are so powerful._\n\nThe sketch below tries to convey this idea. Each quote, once it's made into a vector, is a point in space. Well, in this case it's on a sphere, since OpenAI's embedding vectors, as most others, are normalized to _unit length_. Oh, and the sphere is actually not three-dimensional, rather 1536-dimensional!\n\nSo, in essence, a similarity search in vector space returns the vectors that are closest to the query vector:\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)\n\n**Generation**\n\nGiven a suggestion (a topic or a tentative quote), the search step is performed, and the first returned results (quotes) are fed into an LLM prompt which asks the generative model to invent a new text along the lines of the passed examples _and_ the initial suggestion.\n\n![4_quote_generation](https://user-images.githubusercontent.com/14221764/282440927-d56f36eb-d611-4342-8026-7736edc6f5c9.png)\n\n## Setup\n\nFirst install some required packages:\n\n\n```python\n!pip install --quiet \"cassio>=0.1.3\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nfrom getpass import getpass\nfrom collections import Counter\n\nimport cassio\nfrom cassio.table import MetadataVectorCassandraTable\n\nimport openai\nfrom datasets import load_dataset\n```\n\n## Get DB connection\n\nIn order to connect to your Astra DB through CQL, you need two things:\n- A Token, with role \"Database Administrator\" (it looks like `AstraCS:...`)\n- the database ID (it looks like `3df2a5b6-...`)\n\n    Make sure you have both strings -- which are obtained in the [Astra UI](https://astra.datastax.com) once you sign in. For more information, see here: [database ID](https://awesome-astra.github.io/docs/pages/astra/faq/#where-should-i-find-a-database-identifier) and [Token](https://awesome-astra.github.io/docs/pages/astra/create-token/#c-procedure).\n\nIf you want to _connect to a Cassandra cluster_ (which however must [support](https://cassandra.apache.org/doc/trunk/cassandra/vector-search/overview.html) Vector Search), replace with `cassio.init(session=..., keyspace=...)` with suitable Session and keyspace name for your cluster.\n\n\n```python\nastra_token = getpass(\"Please enter your Astra token ('AstraCS:...')\")\ndatabase_id = input(\"Please enter your database id ('3df2a5b6-...')\")\n```\n\n    Please enter your Astra token ('AstraCS:...') \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n    Please enter your database id ('3df2a5b6-...') 01234567-89ab-dcef-0123-456789abcdef"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: ad7251e5-15ab-4d88-8275-a9faddda9e71
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ad7251e5-15ab-4d88-8275-a9faddda9e71\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ad8a46ab-85c1-4d11-8845-429593ee1cbf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"10db62beb0c3df97d85066800a6b354f27f779763aea3b8ac5436554ae62f8b4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e4ba9e37-e0de-44f1-97d5-ecbe770867de\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"96e9cccddaf4bbbc3eea2886909f129d3b80a095db90b304ebb92206d1f228cc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4171, \"end_char_idx\": 6996, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\n!pip install --quiet \"cassio>=0.1.3\" \"openai>=1.0.0\" datasets\n```\n\n\n```python\nfrom getpass import getpass\nfrom collections import Counter\n\nimport cassio\nfrom cassio.table import MetadataVectorCassandraTable\n\nimport openai\nfrom datasets import load_dataset\n```\n\n## Get DB connection\n\nIn order to connect to your Astra DB through CQL, you need two things:\n- A Token, with role \"Database Administrator\" (it looks like `AstraCS:...`)\n- the database ID (it looks like `3df2a5b6-...`)\n\n    Make sure you have both strings -- which are obtained in the [Astra UI](https://astra.datastax.com) once you sign in. For more information, see here: [database ID](https://awesome-astra.github.io/docs/pages/astra/faq/#where-should-i-find-a-database-identifier) and [Token](https://awesome-astra.github.io/docs/pages/astra/create-token/#c-procedure).\n\nIf you want to _connect to a Cassandra cluster_ (which however must [support](https://cassandra.apache.org/doc/trunk/cassandra/vector-search/overview.html) Vector Search), replace with `cassio.init(session=..., keyspace=...)` with suitable Session and keyspace name for your cluster.\n\n\n```python\nastra_token = getpass(\"Please enter your Astra token ('AstraCS:...')\")\ndatabase_id = input(\"Please enter your database id ('3df2a5b6-...')\")\n```\n\n    Please enter your Astra token ('AstraCS:...') \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n    Please enter your database id ('3df2a5b6-...') 01234567-89ab-dcef-0123-456789abcdef\n\n\n\n```python\ncassio.init(token=astra_token, database_id=database_id)\n```\n\n### Creation of the DB connection\n\nThis is how you create a connection to Astra DB through CQL:\n\n_(Incidentally, you could also use any Cassandra cluster (as long as it provides Vector capabilities), just by [changing the parameters](https://docs.datastax.com/en/developer/python-driver/latest/getting_started/#connecting-to-cassandra) to the following `Cluster` instantiation.)_\n\n### Creation of the Vector Store through CassIO\n\nYou need a table which support vectors and is equipped with metadata. Call it \"philosophers_cassio\":\n\n\n```python\nv_table = MetadataVectorCassandraTable(table=\"philosophers_cassio\", vector_dimension=1536)\n```\n\n## Connect to OpenAI\n\n### Set up your secret key\n\n\n```python\nOPENAI_API_KEY = getpass(\"Please enter your OpenAI API Key: \")\n```\n\n    Please enter your OpenAI API Key:  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: e4ba9e37-e0de-44f1-97d5-ecbe770867de
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e4ba9e37-e0de-44f1-97d5-ecbe770867de\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ad7251e5-15ab-4d88-8275-a9faddda9e71\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c5b85ac5d0b8ac2b90c54bd1b4b38465bc6df8da00c50a98505bc7247fc806c7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4201dc56-c58f-41a3-83a6-627a8165e001\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d21d4aa5e27b2be76776c89aaee9d1fb592d04eba872e3be0a9f6ec03f396635\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5608, \"end_char_idx\": 8538, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\ncassio.init(token=astra_token, database_id=database_id)\n```\n\n### Creation of the DB connection\n\nThis is how you create a connection to Astra DB through CQL:\n\n_(Incidentally, you could also use any Cassandra cluster (as long as it provides Vector capabilities), just by [changing the parameters](https://docs.datastax.com/en/developer/python-driver/latest/getting_started/#connecting-to-cassandra) to the following `Cluster` instantiation.)_\n\n### Creation of the Vector Store through CassIO\n\nYou need a table which support vectors and is equipped with metadata. Call it \"philosophers_cassio\":\n\n\n```python\nv_table = MetadataVectorCassandraTable(table=\"philosophers_cassio\", vector_dimension=1536)\n```\n\n## Connect to OpenAI\n\n### Set up your secret key\n\n\n```python\nOPENAI_API_KEY = getpass(\"Please enter your OpenAI API Key: \")\n```\n\n    Please enter your OpenAI API Key:  \u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\u00b7\n\n\n### A test call for embeddings\n\nQuickly check how one can get the embedding vectors for a list of input texts:\n\n\n```python\nclient = openai.OpenAI(api_key=OPENAI_API_KEY)\nembedding_model_name = \"text-embedding-3-small\"\n\nresult = client.embeddings.create(\n    input=[\n        \"This is a sentence\",\n        \"A second sentence\"\n    ],\n    model=embedding_model_name,\n)\n```\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nprint(f\"len(result.data)              = {len(result.data)}\")\nprint(f\"result.data[1].embedding      = {str(result.data[1].embedding)[:55]}...\")\nprint(f\"len(result.data[1].embedding) = {len(result.data[1].embedding)}\")\n```\n\n    len(result.data)              = 2\n    result.data[1].embedding      = [-0.010821706615388393, 0.001387271680869162, 0.0035479...\n    len(result.data[1].embedding) = 1536\n\n\n## Load quotes into the Vector Store\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 4201dc56-c58f-41a3-83a6-627a8165e001
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4201dc56-c58f-41a3-83a6-627a8165e001\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e4ba9e37-e0de-44f1-97d5-ecbe770867de\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3d9e895b8c6c03a4359005c73e41ee0031e2e7032633227c29ae0cb02b4dd726\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2c4841e6-2cb6-4b20-89dc-d1dbf9491c2f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7a5f4b27ee2c6e7a393c1d8cfbf8278d8cfe88f364df39cf1691eab6e117218d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7407, \"end_char_idx\": 9092, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "## Load quotes into the Vector Store\n\n_Note: the above is the syntax for OpenAI v1.0+. If using previous versions, the code to get the embeddings will look different._\n\n\n```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Insert quotes into vector store\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata planned for later use. Note that the author is added as a metadata field along with the \"tags\" already found with the quote itself.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\n_(Note: for faster execution, Cassandra and CassIO would let you do concurrent inserts, which we don't do here for a more straightforward demo code.)_"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 2c4841e6-2cb6-4b20-89dc-d1dbf9491c2f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2c4841e6-2cb6-4b20-89dc-d1dbf9491c2f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4201dc56-c58f-41a3-83a6-627a8165e001\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c5cbd6cd8354df2186d432871f980e893d087ebaf638c7cc2a5eaaa07e63d439\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"51b97bec-71d4-4444-8ab2-f29b93e562df\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"56239839ff89ff002fd281811addc799de22c6b9d3caf1c6fd33e52904cd7888\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7577, \"end_char_idx\": 11047, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nphilo_dataset = load_dataset(\"datastax/philosopher-quotes\")[\"train\"]\n```\n\nA quick inspection:\n\n\n```python\nprint(\"An example entry:\")\nprint(philo_dataset[16])\n```\n\n    An example entry:\n    {'author': 'aristotle', 'quote': 'Love well, be loved and do something of value.', 'tags': 'love;ethics'}\n\n\nCheck the dataset size:\n\n\n```python\nauthor_count = Counter(entry[\"author\"] for entry in philo_dataset)\nprint(f\"Total: {len(philo_dataset)} quotes. By author:\")\nfor author, count in author_count.most_common():\n    print(f\"    {author:<20}: {count} quotes\")\n```\n\n    Total: 450 quotes. By author:\n        aristotle           : 50 quotes\n        schopenhauer        : 50 quotes\n        spinoza             : 50 quotes\n        hegel               : 50 quotes\n        freud               : 50 quotes\n        nietzsche           : 50 quotes\n        sartre              : 50 quotes\n        plato               : 50 quotes\n        kant                : 50 quotes\n\n\n### Insert quotes into vector store\n\nYou will compute the embeddings for the quotes and save them into the Vector Store, along with the text itself and the metadata planned for later use. Note that the author is added as a metadata field along with the \"tags\" already found with the quote itself.\n\nTo optimize speed and reduce the calls, you'll perform batched calls to the embedding OpenAI service.\n\n_(Note: for faster execution, Cassandra and CassIO would let you do concurrent inserts, which we don't do here for a more straightforward demo code.)_\n\n\n```python\nBATCH_SIZE = 50\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries:\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the rows for insertion\n    print(\"B \", end=\"\")\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = set()\n        author = authors_list[entry_idx]\n        quote = quotes_list[entry_idx]\n        v_table.put(\n            row_id=f\"q_{author}_{entry_idx}\",\n            body_blob=quote,\n            vector=emb_result.embedding,\n            metadata={**{tag: True for tag in tags}, **{\"author\": author}},\n        )\n        print(\"*\", end=\"\")\n    print(f\" done ({len(b_emb_results.data)})\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries:\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    \n    Finished storing entries."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 51b97bec-71d4-4444-8ab2-f29b93e562df
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"51b97bec-71d4-4444-8ab2-f29b93e562df\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2c4841e6-2cb6-4b20-89dc-d1dbf9491c2f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"8d90550b45b02037461caf24de1b246e10d1cddef61039549176249461a05583\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4d6ee676-828b-429d-a07b-3101abd8d40c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3d1b896f62928984106f1753247dcadafa3a18f0ba2c4cfa1c04b79a79d2a576\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9095, \"end_char_idx\": 12581, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nBATCH_SIZE = 50\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries:\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the rows for insertion\n    print(\"B \", end=\"\")\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = set()\n        author = authors_list[entry_idx]\n        quote = quotes_list[entry_idx]\n        v_table.put(\n            row_id=f\"q_{author}_{entry_idx}\",\n            body_blob=quote,\n            vector=emb_result.embedding,\n            metadata={**{tag: True for tag in tags}, **{\"author\": author}},\n        )\n        print(\"*\", end=\"\")\n    print(f\" done ({len(b_emb_results.data)})\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries:\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    B ************************************************** done (50)\n    \n    Finished storing entries.\n\n\n## Use case 1: **quote search engine**\n\nFor the quote-search functionality, you need first to make the input quote into a vector, and then use it to query the store (besides handling the optional metadata into the search call, that is).\n\nEncapsulate the search-engine functionality into a function for ease of re-use:\n\n\n```python\ndef find_quote_and_author(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    metadata = {}\n    if author:\n        metadata[\"author\"] = author\n    if tags:\n        for tag in tags:\n            metadata[tag] = True\n    #\n    results = v_table.ann_search(\n        query_vector,\n        n=n,\n        metadata=metadata,\n    )\n    return [\n        (result[\"body_blob\"], result[\"metadata\"][\"author\"])\n        for result in results\n    ]\n```\n\n### Putting search to test\n\nPassing just a quote:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 4d6ee676-828b-429d-a07b-3101abd8d40c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4d6ee676-828b-429d-a07b-3101abd8d40c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"51b97bec-71d4-4444-8ab2-f29b93e562df\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e5f0820022b91c403566c4d1e598952f37aa7625e16a4a70209c2449a6b96330\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"32ba905b-8283-4755-bec3-be8b083293c1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"674d2ce49e35837bb7699f4d5ccdd387f1f9d1cf740937254d492f9ceb101336\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11050, \"end_char_idx\": 13363, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "## Use case 1: **quote search engine**\n\nFor the quote-search functionality, you need first to make the input quote into a vector, and then use it to query the store (besides handling the optional metadata into the search call, that is).\n\nEncapsulate the search-engine functionality into a function for ease of re-use:\n\n\n```python\ndef find_quote_and_author(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    metadata = {}\n    if author:\n        metadata[\"author\"] = author\n    if tags:\n        for tag in tags:\n            metadata[tag] = True\n    #\n    results = v_table.ann_search(\n        query_vector,\n        n=n,\n        metadata=metadata,\n    )\n    return [\n        (result[\"body_blob\"], result[\"metadata\"][\"author\"])\n        for result in results\n    ]\n```\n\n### Putting search to test\n\nPassing just a quote:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nSearch constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('Mankind will never see an end of trouble until lovers of wisdom come to hold political power, or the holders of power become lovers of wisdom',\n      'plato'),\n     ('Everything the State says is a lie, and everything it has it has stolen.',\n      'nietzsche')]"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 32ba905b-8283-4755-bec3-be8b083293c1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"32ba905b-8283-4755-bec3-be8b083293c1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4d6ee676-828b-429d-a07b-3101abd8d40c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2e0235d8613999234e13485a56665ee4b7159cb978e5b0aaf8f8f24739770aae\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7c0134c7-4ed5-4fb1-b8c7-061017d3d0ce\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"46b54953fac4856142fa32a9ed617716bebde4b7627df54619e571f59a013c1d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11995, \"end_char_idx\": 14557, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\nSearch restricted to an author:\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nSearch constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('Mankind will never see an end of trouble until lovers of wisdom come to hold political power, or the holders of power become lovers of wisdom',\n      'plato'),\n     ('Everything the State says is a lie, and everything it has it has stolen.',\n      'nietzsche')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"distance\" between the query and each result, and then set a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results:\n\n_Note (for the mathematically inclined): this \"distance\" is exactly the cosine similarity between the vectors, i.e. the scalar product divided by the product of the norms of the two vectors. As such, it is a number ranging from -1 to +1, where -1 is for exactly opposite-facing vectors and +1 for identically-oriented vectors. Elsewhere (e.g. in the \"CQL\" counterpart of this demo) you would get a rescaling of this quantity to fit the [0, 1] interval, which means the resulting numerical values and adequate thresholds there are transformed accordingly._"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 7c0134c7-4ed5-4fb1-b8c7-061017d3d0ce
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7c0134c7-4ed5-4fb1-b8c7-061017d3d0ce\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"32ba905b-8283-4755-bec3-be8b083293c1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"929c6445853f9b9a45ff8952594bdeb3f34dd71b9215ba86808e68420c84d247\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"98a0ec39-0a4a-430f-9545-6a9464aa2318\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"89d48b7eaa67e5c817c7687ebd38cd8cde2bfd848b371004be805ff78708f4cd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12917, \"end_char_idx\": 16194, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "Search constrained to a tag (out of those saved earlier with the quotes):\n\n\n```python\nfind_quote_and_author(\"We struggle all our life for nothing\", 2, tags=[\"politics\"])\n```\n\n\n\n\n    [('Mankind will never see an end of trouble until lovers of wisdom come to hold political power, or the holders of power become lovers of wisdom',\n      'plato'),\n     ('Everything the State says is a lie, and everything it has it has stolen.',\n      'nietzsche')]\n\n\n\n### Cutting out irrelevant results\n\nThe vector similarity search generally returns the vectors that are closest to the query, even if that means results that might be somewhat irrelevant if there's nothing better.\n\nTo keep this issue under control, you can get the actual \"distance\" between the query and each result, and then set a cutoff on it, effectively discarding results that are beyond that threshold.\nTuning this threshold correctly is not an easy problem: here, we'll just show you the way.\n\nTo get a feeling on how this works, try the following query and play with the choice of quote and threshold to compare the results:\n\n_Note (for the mathematically inclined): this \"distance\" is exactly the cosine similarity between the vectors, i.e. the scalar product divided by the product of the norms of the two vectors. As such, it is a number ranging from -1 to +1, where -1 is for exactly opposite-facing vectors and +1 for identically-oriented vectors. Elsewhere (e.g. in the \"CQL\" counterpart of this demo) you would get a rescaling of this quantity to fit the [0, 1] interval, which means the resulting numerical values and adequate thresholds there are transformed accordingly._\n\n\n```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nmetric_threshold = 0.84\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\nresults = list(v_table.metric_ann_search(\n    quote_vector,\n    n=8,\n    metric=\"cos\",\n    metric_threshold=metric_threshold,\n))\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, result in enumerate(results):\n    print(f\"    {idx}. [distance={result['distance']:.3f}] \\\"{result['body_blob'][:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [distance=0.855] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [distance=0.843] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [distance=0.841] \"At his best, man is the noblest of all animals; separated from law and...\"\n\n\n## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task.\n\n\n```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 98a0ec39-0a4a-430f-9545-6a9464aa2318
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"98a0ec39-0a4a-430f-9545-6a9464aa2318\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7c0134c7-4ed5-4fb1-b8c7-061017d3d0ce\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1fd26059dcf9c67dbcdafec81d0314d17a32a98c0e3075f88520704673872abc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5b94e66d-6580-4055-9d35-1cddc9b23385\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c188a6e724b939f89487e1ae7ca4bf7a3a9f415d819c43a28f76446a1fb980d7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14560, \"end_char_idx\": 17411, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nquote = \"Animals are our equals.\"\n# quote = \"Be good.\"\n# quote = \"This teapot is strange.\"\n\nmetric_threshold = 0.84\n\nquote_vector = client.embeddings.create(\n    input=[quote],\n    model=embedding_model_name,\n).data[0].embedding\n\nresults = list(v_table.metric_ann_search(\n    quote_vector,\n    n=8,\n    metric=\"cos\",\n    metric_threshold=metric_threshold,\n))\n\nprint(f\"{len(results)} quotes within the threshold:\")\nfor idx, result in enumerate(results):\n    print(f\"    {idx}. [distance={result['distance']:.3f}] \\\"{result['body_blob'][:70]}...\\\"\")\n```\n\n    3 quotes within the threshold:\n        0. [distance=0.855] \"The assumption that animals are without rights, and the illusion that ...\"\n        1. [distance=0.843] \"Animals are in possession of themselves; their soul is in possession o...\"\n        2. [distance=0.841] \"At his best, man is the noblest of all animals; separated from law and...\"\n\n\n## Use case 2: **quote generator**\n\nFor this task you need another component from OpenAI, namely an LLM to generate the quote for us (based on input obtained by querying the Vector Store).\n\nYou also need a template for the prompt that will be filled for the generate-quote LLM completion task.\n\n\n```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):\n\n\n```python\ndef generate_quote(topic, n=2, author=None, tags=None):\n    quotes = find_quote_and_author(query_quote=topic, n=n, author=author, tags=tags)\n    if quotes:\n        prompt = generation_prompt_template.format(\n            topic=topic,\n            examples=\"\\n\".join(f\"  - {quote[0]}\" for quote in quotes),\n        )\n        # a little logging:\n        print(\"** quotes found:\")\n        for q, a in quotes:\n            print(f\"**    - {q} ({a})\")\n        print(\"** end of logging\")\n        #\n        response = client.chat.completions.create(\n            model=completion_model_name,\n            messages=[{\"role\": \"user\", \"content\": prompt}],\n            temperature=0.7,\n            max_tokens=320,\n        )\n        return response.choices[0].message.content.replace('\"', '').strip()\n    else:\n        print(\"** no quotes found.\")\n        return None\n```\n\n_Note: similar to the case of the embedding computation, the code for the Chat Completion API would be slightly different for OpenAI prior to v1.0._\n\n#### Putting quote generation to test\n\nJust passing a text (a \"quote\", but one can actually just suggest a topic since its vector embedding will still end up at the right place in the vector space):"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 5b94e66d-6580-4055-9d35-1cddc9b23385
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5b94e66d-6580-4055-9d35-1cddc9b23385\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"98a0ec39-0a4a-430f-9545-6a9464aa2318\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1520c044285032ea5a358010c8e0eeab28a8cddbf5ba00316720d6118c17394b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4dfc21b6-f24b-471d-9daa-1e4f9a974791\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"28f01cd6aa2437d3c28608fe26670f51cd613835fcffabd190b11f5eb4a8c835\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15768, \"end_char_idx\": 19208, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\ncompletion_model_name = \"gpt-3.5-turbo\"\n\ngeneration_prompt_template = \"\"\"\"Generate a single short philosophical quote on the given topic,\nsimilar in spirit and form to the provided actual example quotes.\nDo not exceed 20-30 words in your quote.\n\nREFERENCE TOPIC: \"{topic}\"\n\nACTUAL EXAMPLES:\n{examples}\n\"\"\"\n```\n\nLike for search, this functionality is best wrapped into a handy function (which internally uses search):\n\n\n```python\ndef generate_quote(topic, n=2, author=None, tags=None):\n    quotes = find_quote_and_author(query_quote=topic, n=n, author=author, tags=tags)\n    if quotes:\n        prompt = generation_prompt_template.format(\n            topic=topic,\n            examples=\"\\n\".join(f\"  - {quote[0]}\" for quote in quotes),\n        )\n        # a little logging:\n        print(\"** quotes found:\")\n        for q, a in quotes:\n            print(f\"**    - {q} ({a})\")\n        print(\"** end of logging\")\n        #\n        response = client.chat.completions.create(\n            model=completion_model_name,\n            messages=[{\"role\": \"user\", \"content\": prompt}],\n            temperature=0.7,\n            max_tokens=320,\n        )\n        return response.choices[0].message.content.replace('\"', '').strip()\n    else:\n        print(\"** no quotes found.\")\n        return None\n```\n\n_Note: similar to the case of the embedding computation, the code for the Chat Completion API would be slightly different for OpenAI prior to v1.0._\n\n#### Putting quote generation to test\n\nJust passing a text (a \"quote\", but one can actually just suggest a topic since its vector embedding will still end up at the right place in the vector space):\n\n\n```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    Virtuous politics purifies society, while corrupt politics breeds chaos and decay.\n\n\nUse inspiration from just a single philosopher:\n\n\n```python\nq_topic = generate_quote(\"animals\", author=\"schopenhauer\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Because Christian morality leaves animals out of account, they are at once outlawed in philosophical morals; they are mere 'things,' mere means to any ends whatsoever. They can therefore be used for vivisection, hunting, coursing, bullfights, and horse racing, and can be whipped to death as they struggle along with heavy carts of stone. Shame on such a morality that is worthy of pariahs, and that fails to recognize the eternal essence that exists in every living thing, and shines forth with inscrutable significance from all eyes that see the sun! (schopenhauer)\n    **    - The assumption that animals are without rights, and the illusion that our treatment of them has no moral significance, is a positively outrageous example of Western crudity and barbarity. Universal compassion is the only guarantee of morality. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    The true measure of humanity lies not in our dominion over animals, but in our ability to show compassion and respect for all living beings."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 4dfc21b6-f24b-471d-9daa-1e4f9a974791
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4dfc21b6-f24b-471d-9daa-1e4f9a974791\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5b94e66d-6580-4055-9d35-1cddc9b23385\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"572205a30c09ff140fbca2a7f25b0da8acc2d55af121297633fe50672e167e13\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"44007ae2-eef4-4ac9-9527-6c670af8cc5e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c26e01f72c6f2863c881361edd6a6c855ab22d8284c6cc1a8dae242198d2c4cf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17414, \"end_char_idx\": 20295, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nq_topic = generate_quote(\"politics and virtue\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Happiness is the reward of virtue. (aristotle)\n    **    - Our moral virtues benefit mainly other people; intellectual virtues, on the other hand, benefit primarily ourselves; therefore the former make us universally popular, the latter unpopular. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    Virtuous politics purifies society, while corrupt politics breeds chaos and decay.\n\n\nUse inspiration from just a single philosopher:\n\n\n```python\nq_topic = generate_quote(\"animals\", author=\"schopenhauer\")\nprint(\"\\nA new generated quote:\")\nprint(q_topic)\n```\n\n    ** quotes found:\n    **    - Because Christian morality leaves animals out of account, they are at once outlawed in philosophical morals; they are mere 'things,' mere means to any ends whatsoever. They can therefore be used for vivisection, hunting, coursing, bullfights, and horse racing, and can be whipped to death as they struggle along with heavy carts of stone. Shame on such a morality that is worthy of pariahs, and that fails to recognize the eternal essence that exists in every living thing, and shines forth with inscrutable significance from all eyes that see the sun! (schopenhauer)\n    **    - The assumption that animals are without rights, and the illusion that our treatment of them has no moral significance, is a positively outrageous example of Western crudity and barbarity. Universal compassion is the only guarantee of morality. (schopenhauer)\n    ** end of logging\n    \n    A new generated quote:\n    The true measure of humanity lies not in our dominion over animals, but in our ability to show compassion and respect for all living beings.\n\n\n## (Optional) **Partitioning**\n\nThere's an interesting topic to examine before completing this quickstart. While, generally, tags and quotes can be in any relationship (e.g. a quote having multiple tags), _authors_ are effectively an exact grouping (they define a \"disjoint partitioning\" on the set of quotes): each quote has exactly one author (for us, at least).\n\nNow, suppose you know in advance your application will usually (or always) run queries on a _single author_. Then you can take full advantage of the underlying database structure: if you group quotes in **partitions** (one per author), vector queries on just an author will use less resources and return much faster.\n\nWe'll not dive into the details here, which have to do with the Cassandra storage internals: the important message is that **if your queries are run within a group, consider partitioning accordingly to boost performance**.\n\nYou'll now see this choice in action.\n\nFirst, you need a different table abstraction from CassIO:\n\n\n```python\nfrom cassio.table import ClusteredMetadataVectorCassandraTable\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 44007ae2-eef4-4ac9-9527-6c670af8cc5e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"44007ae2-eef4-4ac9-9527-6c670af8cc5e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4dfc21b6-f24b-471d-9daa-1e4f9a974791\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"39cf7c9d6d0b297719c69bbd675abb83a181a9d317136a1af41b50f85b9b4731\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9844bfb7-5947-4955-84d2-d98eff6d20a0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"06cf7088182946ccae7ef6824b5c7de4c4cc7556832bfe7598b65f1ca1a76eab\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19211, \"end_char_idx\": 21300, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "## (Optional) **Partitioning**\n\nThere's an interesting topic to examine before completing this quickstart. While, generally, tags and quotes can be in any relationship (e.g. a quote having multiple tags), _authors_ are effectively an exact grouping (they define a \"disjoint partitioning\" on the set of quotes): each quote has exactly one author (for us, at least).\n\nNow, suppose you know in advance your application will usually (or always) run queries on a _single author_. Then you can take full advantage of the underlying database structure: if you group quotes in **partitions** (one per author), vector queries on just an author will use less resources and return much faster.\n\nWe'll not dive into the details here, which have to do with the Cassandra storage internals: the important message is that **if your queries are run within a group, consider partitioning accordingly to boost performance**.\n\nYou'll now see this choice in action.\n\nFirst, you need a different table abstraction from CassIO:\n\n\n```python\nfrom cassio.table import ClusteredMetadataVectorCassandraTable\n```\n\n\n```python\nv_table_partitioned = ClusteredMetadataVectorCassandraTable(table=\"philosophers_cassio_partitioned\", vector_dimension=1536)\n```\n\nNow repeat the compute-embeddings-and-insert step on the new table.\n\nCompared to what you have seen earlier, there is a crucial difference in that now the quote's author is stored as the _partition id_ for the inserted row, instead of being added to the catch-all \"metadata\" dictionary.\n\nWhile you are at it, by way of demonstration, you will insert all quotes by a given author _concurrently_: with CassIO, this is done by usng the asynchronous `put_async` method for each quote, collecting the resulting list of `Future` objects, and calling the `result()` method on them all afterwards, to ensure they all have executed. Cassandra / Astra DB well supports a high degree of concurrency in I/O operations.\n\n_(Note: one could have cached the embeddings computed previously to save a few API tokens -- here, however, we wanted to keep the code easier to inspect.)_"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 9844bfb7-5947-4955-84d2-d98eff6d20a0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"9844bfb7-5947-4955-84d2-d98eff6d20a0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"44007ae2-eef4-4ac9-9527-6c670af8cc5e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"df9f374bc5ae43309d4f8bfdbe82dd35f2a864770dcf3d676d6b811699f50b2e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"184e51ef-55f8-4981-b390-804fb047ef5f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b8d19c4159f80dc3a9c720af1ea488944a32a179071927843ff77b665d6dcd38\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20219, \"end_char_idx\": 23002, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "```python\nfrom cassio.table import ClusteredMetadataVectorCassandraTable\n```\n\n\n```python\nv_table_partitioned = ClusteredMetadataVectorCassandraTable(table=\"philosophers_cassio_partitioned\", vector_dimension=1536)\n```\n\nNow repeat the compute-embeddings-and-insert step on the new table.\n\nCompared to what you have seen earlier, there is a crucial difference in that now the quote's author is stored as the _partition id_ for the inserted row, instead of being added to the catch-all \"metadata\" dictionary.\n\nWhile you are at it, by way of demonstration, you will insert all quotes by a given author _concurrently_: with CassIO, this is done by usng the asynchronous `put_async` method for each quote, collecting the resulting list of `Future` objects, and calling the `result()` method on them all afterwards, to ensure they all have executed. Cassandra / Astra DB well supports a high degree of concurrency in I/O operations.\n\n_(Note: one could have cached the embeddings computed previously to save a few API tokens -- here, however, we wanted to keep the code easier to inspect.)_\n\n\n```python\nBATCH_SIZE = 50\n\nnum_batches = ((len(philo_dataset) + BATCH_SIZE - 1) // BATCH_SIZE)\n\nquotes_list = philo_dataset[\"quote\"]\nauthors_list = philo_dataset[\"author\"]\ntags_list = philo_dataset[\"tags\"]\n\nprint(\"Starting to store entries:\")\nfor batch_i in range(num_batches):\n    b_start = batch_i * BATCH_SIZE\n    b_end = (batch_i + 1) * BATCH_SIZE\n    # compute the embedding vectors for this batch\n    b_emb_results = client.embeddings.create(\n        input=quotes_list[b_start : b_end],\n        model=embedding_model_name,\n    )\n    # prepare the rows for insertion\n    futures = []\n    print(\"B \", end=\"\")\n    for entry_idx, emb_result in zip(range(b_start, b_end), b_emb_results.data):\n        if tags_list[entry_idx]:\n            tags = {\n                tag\n                for tag in tags_list[entry_idx].split(\";\")\n            }\n        else:\n            tags = set()\n        author = authors_list[entry_idx]\n        quote = quotes_list[entry_idx]\n        futures.append(v_table_partitioned.put_async(\n            partition_id=author,\n            row_id=f\"q_{author}_{entry_idx}\",\n            body_blob=quote,\n            vector=emb_result.embedding,\n            metadata={tag: True for tag in tags},\n        ))\n    #\n    for future in futures:\n        future.result()\n    #\n    print(f\" done ({len(b_emb_results.data)})\")\n\nprint(\"\\nFinished storing entries.\")\n```\n\n    Starting to store entries:\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    B  done (50)\n    \n    Finished storing entries.\n\n\nWith this new table, the similarity search changes accordingly (note the arguments to `ann_search`):"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 184e51ef-55f8-4981-b390-804fb047ef5f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb", "title": "Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL", "tokens": 6051, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"184e51ef-55f8-4981-b390-804fb047ef5f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9cd2cbcf-42f8-4819-bfca-1ee231c43303\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"723f825cd19bd5d01794b8111cf19db861b8b27b9247f5207316ec1dc1f05a13\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9844bfb7-5947-4955-84d2-d98eff6d20a0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb\", \"title\": \"Philosophy with Vector Embeddings, OpenAI and Cassandra / Astra DB through CQL\", \"tokens\": 6051, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e9594886c16e10c019bddf58f236ff9c58aebc9c1c421e64605d29d14ebdc121\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22902, \"end_char_idx\": 26172, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "ref_doc_id": "9cd2cbcf-42f8-4819-bfca-1ee231c43303", "chroma:document": "With this new table, the similarity search changes accordingly (note the arguments to `ann_search`):\n\n\n```python\ndef find_quote_and_author_p(query_quote, n, author=None, tags=None):\n    query_vector = client.embeddings.create(\n        input=[query_quote],\n        model=embedding_model_name,\n    ).data[0].embedding\n    metadata = {}\n    partition_id = None\n    if author:\n        partition_id = author\n    if tags:\n        for tag in tags:\n            metadata[tag] = True\n    #\n    results = v_table_partitioned.ann_search(\n        query_vector,\n        n=n,\n        partition_id=partition_id,\n        metadata=metadata,\n    )\n    return [\n        (result[\"body_blob\"], result[\"partition_id\"])\n        for result in results\n    ]\n```\n\nThat's it: the new table still supports the \"generic\" similarity searches all right ...\n\n\n```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 3)\n```\n\n\n\n\n    [('Life to the great majority is only a constant struggle for mere existence, with the certainty of losing it at last.',\n      'schopenhauer'),\n     ('We give up leisure in order that we may have leisure, just as we go to war in order that we may have peace.',\n      'aristotle'),\n     ('Perhaps the gods are kind to us, by making life more disagreeable as we grow older. In the end death seems less intolerable than the manifold burdens we carry',\n      'freud')]\n\n\n\n... but it's when an author is specified that you would notice a _huge_ performance advantage:\n\n\n```python\nfind_quote_and_author_p(\"We struggle all our life for nothing\", 2, author=\"nietzsche\")\n```\n\n\n\n\n    [('To live is to suffer, to survive is to find some meaning in the suffering.',\n      'nietzsche'),\n     ('What makes us heroic?--Confronting simultaneously our supreme suffering and our supreme hope.',\n      'nietzsche')]\n\n\n\nWell, you _would_ notice a performance gain, if you had a realistic-size dataset. In this demo, with a few tens of entries, there's no noticeable difference -- but you get the idea.\n\n## Conclusion\n\nCongratulations! You have learned how to use OpenAI for vector embeddings and Cassandra / Astra DB through CQL for storage in order to build a sophisticated philosophical search engine and quote generator.\n\nThis example used [CassIO](https://cassio.org) to interface with the Vector Store - but this is not the only choice. Check the [README](https://github.com/openai/openai-cookbook/tree/main/examples/vector_databases/cassandra_astradb) for other options and integration with popular frameworks.\n\nTo find out more on how Astra DB's Vector Search capabilities can be a key ingredient in your ML/GenAI applications, visit [Astra DB](https://docs.datastax.com/en/astra/home/astra.html)'s web page on the topic.\n\n## Cleanup\n\nIf you want to remove all resources used for this demo, run this cell (_warning: this will delete the tables and the data inserted in them!_):\n\n\n```python\n# we peek at CassIO's config to get a direct handle to the DB connection\nsession = cassio.config.resolve_session()\nkeyspace = cassio.config.resolve_keyspace()\n\nsession.execute(f\"DROP TABLE IF EXISTS {keyspace}.philosophers_cassio;\")\nsession.execute(f\"DROP TABLE IF EXISTS {keyspace}.philosophers_cassio_partitioned;\")\n```\n\n\n\n\n    <cassandra.cluster.ResultSet at 0x7fdcc42e8f10>"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: f010eb03-95b9-4445-9512-141268082ce4
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/README.ipynb", "title": "RAG with Astra DB and Cassandra", "tokens": 669, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f010eb03-95b9-4445-9512-141268082ce4\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/README.ipynb\", \"title\": \"RAG with Astra DB and Cassandra\", \"tokens\": 669, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ba4aabc-fc39-4147-9cf1-b8a169cc1069\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/README.ipynb\", \"title\": \"RAG with Astra DB and Cassandra\", \"tokens\": 669, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a90d81b31b38a6276c58f70dec4f8a6484ba5b0f1488314322d7da05d6bcd1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2631, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ba4aabc-fc39-4147-9cf1-b8a169cc1069", "doc_id": "6ba4aabc-fc39-4147-9cf1-b8a169cc1069", "ref_doc_id": "6ba4aabc-fc39-4147-9cf1-b8a169cc1069", "chroma:document": "# RAG with Astra DB and Cassandra\n\nThe demos in this directory show how to use the Vector\nSearch capabilities available today in **DataStax Astra DB**, a serverless\nDatabase-as-a-Service built on Apache Cassandra\u00ae.\n\nThese example notebooks demonstrate implementation of\nthe same GenAI standard RAG workload with different libraries and APIs.\n\nTo use [Astra DB](https://docs.datastax.com/en/astra/home/astra.html)\nwith its HTTP API interface, head to the \"AstraPy\" notebook (`astrapy`\nis the Python client to interact with the database).\n\nIf you prefer CQL access to the database (either with\n[Astra DB](https://docs.datastax.com/en/astra-serverless/docs/vector-search/overview.html)\nor a Cassandra cluster\n[supporting vector search](https://cassandra.apache.org/doc/trunk/cassandra/vector-search/overview.html)),\ncheck the \"CQL\" or \"CassIO\" notebooks -- they differ in the level of abstraction you get to work at.\n\nIf you want to know more about Astra DB and its Vector Search capabilities,\nhead over to [datastax.com](https://docs.datastax.com/en/astra/home/astra.html).\n\n### Example notebooks\n\nThe following examples show how easily OpenAI and DataStax Astra DB can\nwork together to power vector-based AI applications. You can run them either\nwith your local Jupyter engine or as Colab notebooks:\n\n| Use case | Target database | Framework | Notebook | Google Colab |\n| -------- | --------------- | --------- | -------- | ------------ |\n| Search/generate quotes | Astra DB | AstraPy | [Notebook](./Philosophical_Quotes_AstraPy.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_AstraPy.ipynb) |\n| Search/generate quotes | Cassandra / Astra DB through CQL | CassIO | [Notebook](./Philosophical_Quotes_cassIO.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_cassIO.ipynb) |\n| Search/generate quotes | Cassandra / Astra DB through CQL | Plain Cassandra language | [Notebook](./Philosophical_Quotes_CQL.ipynb) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/openai/openai-cookbook/blob/main/examples/vector_databases/cassandra_astradb/Philosophical_Quotes_CQL.ipynb) |\n\n### Vector similarity, visual representation\n\n![3_vector_space](https://user-images.githubusercontent.com/14221764/262321363-c8c625c1-8be9-450e-8c68-b1ed518f990d.png)"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 0c0f0f2c-7575-4bd6-868b-7cfa2cb2e669
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0c0f0f2c-7575-4bd6-868b-7cfa2cb2e669\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3cd8590a-da9b-494c-82ae-f0ec67b15f73\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"de6fcb349e7fc5d729b2ec4d246c9b47eb380869b5124a394c53b81a7a83c9c9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3615, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "# Using MyScale for Embeddings Search\n\nThis notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.\n\n### What is a Vector Database\n\nA vector database is a database made to store, manage and search embedding vectors. The use of embeddings to encode unstructured data (text, audio, video and more) as vectors for consumption by machine-learning models has exploded in recent years, due to the increasing effectiveness of AI in solving use cases involving natural language, image recognition and other unstructured forms of data. Vector databases have emerged as an effective solution for enterprises to deliver and scale these use cases.\n\n### Why use a Vector Database\n\nVector databases enable enterprises to take many of the embeddings use cases we've shared in this repo (question and answering, chatbot and recommendation services, for example), and make use of them in a secure, scalable environment. Many of our customers make embeddings solve their problems at small scale but performance and security hold them back from going into production - we see vector databases as a key component in solving that, and in this guide we'll walk through the basics of embedding text data, storing it in a vector database and using it for semantic search.\n\n\n### Demo Flow\nThe demo flow is:\n- **Setup**: Import packages and set any required variables\n- **Load data**: Load a dataset and embed it using OpenAI embeddings\n- **MyScale**\n    - *Setup*: Set up the MyScale Python client. For more details go [here](https://docs.myscale.com/en/python-client/)\n    - *Index Data*: We'll create a table and index it for __content__.\n    - *Search Data*: Run a few example queries with various goals in mind.\n\nOnce you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings.\n\n## Setup\n\nImport the required libraries and set the embedding model that we'd like to use.\n\n\n```python\n# We'll need to install the MyScale client\n!pip install clickhouse-connect\n\n#Install wget to pull zip file\n!pip install wget\n```\n\n\n```python\nimport openai\n\nfrom typing import List, Iterator\nimport pandas as pd\nimport numpy as np\nimport os\nimport wget\nfrom ast import literal_eval\n\n# MyScale's client library for Python\nimport clickhouse_connect\n\n# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\nEMBEDDING_MODEL = \"text-embedding-3-small\"\n\n# Ignore unclosed SSL socket warnings - optional in case you get these errors\nimport warnings\n\nwarnings.filterwarnings(action=\"ignore\", message=\"unclosed\", category=ResourceWarning)\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n```\n\n## Load data\n\nIn this section we'll load embedded data that we've prepared previous to this session.\n\n\n```python\nembeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\n\n```python\nimport zipfile\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"../data\")\n```\n\n\n```python\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\n```\n\n\n```python\narticle_df.head()\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 3cd8590a-da9b-494c-82ae-f0ec67b15f73
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3cd8590a-da9b-494c-82ae-f0ec67b15f73\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0c0f0f2c-7575-4bd6-868b-7cfa2cb2e669\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e471cb199bbb6499ed8c87a093f68be47521ee8a5f800c81b13f25d00f7df59b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6b026e19-3cc9-47a0-9fbd-836b6bb9b4d8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"55d75d734c6c465942a4bbe99c9e2550ca1dde90d5af183d906f3fc23ae149c3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2269, \"end_char_idx\": 4631, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "```python\n# We'll need to install the MyScale client\n!pip install clickhouse-connect\n\n#Install wget to pull zip file\n!pip install wget\n```\n\n\n```python\nimport openai\n\nfrom typing import List, Iterator\nimport pandas as pd\nimport numpy as np\nimport os\nimport wget\nfrom ast import literal_eval\n\n# MyScale's client library for Python\nimport clickhouse_connect\n\n# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\nEMBEDDING_MODEL = \"text-embedding-3-small\"\n\n# Ignore unclosed SSL socket warnings - optional in case you get these errors\nimport warnings\n\nwarnings.filterwarnings(action=\"ignore\", message=\"unclosed\", category=ResourceWarning)\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning) \n```\n\n## Load data\n\nIn this section we'll load embedded data that we've prepared previous to this session.\n\n\n```python\nembeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\n\n```python\nimport zipfile\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"../data\")\n```\n\n\n```python\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\n```\n\n\n```python\narticle_df.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.)"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 6b026e19-3cc9-47a0-9fbd-836b6bb9b4d8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6b026e19-3cc9-47a0-9fbd-836b6bb9b4d8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3cd8590a-da9b-494c-82ae-f0ec67b15f73\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a3e97206888675368b528c22d59c7fef64d48483dff473c9ba0afe3b73012aab\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4fb7c8b4-f379-4320-b70e-82a844bce89a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0eb9f1fbcad7c87e0ae1cc80d55f22c888212d66f0901dbcd88725f848899c3c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3484, \"end_char_idx\": 5662, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "```python\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\n```\n\n\n```python\narticle_df.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>url</th>\n      <th>title</th>\n      <th>text</th>\n      <th>title_vector</th>\n      <th>content_vector</th>\n      <th>vector_id</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>https://simple.wikipedia.org/wiki/April</td>\n      <td>April</td>\n      <td>April is the fourth month of the year in the J...</td>\n      <td>[0.001009464613161981, -0.020700545981526375, ...</td>\n      <td>[-0.011253940872848034, -0.013491976074874401,...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>https://simple.wikipedia.org/wiki/August</td>\n      <td>August</td>\n      <td>August (Aug.) is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 4fb7c8b4-f379-4320-b70e-82a844bce89a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4fb7c8b4-f379-4320-b70e-82a844bce89a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6b026e19-3cc9-47a0-9fbd-836b6bb9b4d8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9c2e89c85f6d1531055a4490587a8ee7a4f4b55ba5b4847f5b570539af44ab8b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6dc5fc47-d853-427d-b779-cc8c99434c94\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"838f0cc3d0ead1c682db56092fdb0a44134572846028a508c3f613d3bf701d6e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4632, \"end_char_idx\": 6830, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "is the eighth month of the year ...</td>\n      <td>[0.0009286514250561595, 0.000820168002974242, ...</td>\n      <td>[0.0003609954728744924, 0.007262262050062418, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>6</td>\n      <td>https://simple.wikipedia.org/wiki/Art</td>\n      <td>Art</td>\n      <td>Art is a creative activity that expresses imag...</td>\n      <td>[0.003393713850528002, 0.0061537534929811954, ...</td>\n      <td>[-0.004959689453244209, 0.015772193670272827, ...</td>\n      <td>2</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>8</td>\n      <td>https://simple.wikipedia.org/wiki/A</td>\n      <td>A</td>\n      <td>A or a is the first letter of the English alph...</td>\n      <td>[0.0153952119871974, -0.013759135268628597, 0....</td>\n      <td>[0.024894846603274345, -0.022186409682035446, ...</td>\n      <td>3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9</td>\n      <td>https://simple.wikipedia.org/wiki/Air</td>\n      <td>Air</td>\n      <td>Air refers to the Earth's atmosphere. Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\n# Read vectors from strings back into a list\narticle_df['title_vector'] = article_df.title_vector.apply(literal_eval)\narticle_df['content_vector'] = article_df.content_vector.apply(literal_eval)\n\n# Set vector_id to be a string\narticle_df['vector_id'] = article_df['vector_id'].apply(str)\n```\n\n\n```python\narticle_df.info(show_counts=True)\n```\n\n    <class 'pandas.core.frame.DataFrame'>\n    RangeIndex: 25000 entries, 0 to 24999\n    Data columns (total 7 columns):\n     #   Column          Non-Null Count  Dtype \n    ---  ------          --------------  ----- \n     0   id              25000 non-null  int64 \n     1   url             25000 non-null  object\n     2   title           25000 non-null  object\n     3   text            25000 non-null  object\n     4   title_vector    25000 non-null  object\n     5   content_vector  25000 non-null  object\n     6   vector_id       25000 non-null  object\n    dtypes: int64(1), object(6)\n    memory usage: 1.3+ MB"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 6dc5fc47-d853-427d-b779-cc8c99434c94
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6dc5fc47-d853-427d-b779-cc8c99434c94\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4fb7c8b4-f379-4320-b70e-82a844bce89a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a56655af6402a8182db44ddf3b0cebb90c6b748b9983e5bd48449b2e03e07f04\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"089bf370-91cd-4672-aec5-a37690ed7b8b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"05f076a3140590c1ce448473600f589e8e8bd58f97fcb49d5f379d071502612b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5663, \"end_char_idx\": 8125, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "Air is a...</td>\n      <td>[0.02224554680287838, -0.02044147066771984, -0...</td>\n      <td>[0.021524671465158463, 0.018522677943110466, -...</td>\n      <td>4</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\n# Read vectors from strings back into a list\narticle_df['title_vector'] = article_df.title_vector.apply(literal_eval)\narticle_df['content_vector'] = article_df.content_vector.apply(literal_eval)\n\n# Set vector_id to be a string\narticle_df['vector_id'] = article_df['vector_id'].apply(str)\n```\n\n\n```python\narticle_df.info(show_counts=True)\n```\n\n    <class 'pandas.core.frame.DataFrame'>\n    RangeIndex: 25000 entries, 0 to 24999\n    Data columns (total 7 columns):\n     #   Column          Non-Null Count  Dtype \n    ---  ------          --------------  ----- \n     0   id              25000 non-null  int64 \n     1   url             25000 non-null  object\n     2   title           25000 non-null  object\n     3   text            25000 non-null  object\n     4   title_vector    25000 non-null  object\n     5   content_vector  25000 non-null  object\n     6   vector_id       25000 non-null  object\n    dtypes: int64(1), object(6)\n    memory usage: 1.3+ MB\n\n\n## MyScale\nThe next vector database we'll consider is [MyScale](https://myscale.com).\n\n[MyScale](https://myscale.com) is a database built on Clickhouse that combines vector search and SQL analytics to offer a high-performance, streamlined, and fully managed experience. It's designed to facilitate joint queries and analyses on both structured and vector data, with comprehensive SQL support for all data processing.\n\nDeploy and execute vector search with SQL on your cluster within two minutes by using [MyScale Console](https://console.myscale.com).\n\n### Connect to MyScale\n\nFollow the [connections details](https://docs.myscale.com/en/cluster-management/) section to retrieve the cluster host, username, and password information from the MyScale console, and use it to create a connection to your cluster as shown below:\n\n\n```python\n# initialize client\nclient = clickhouse_connect.get_client(host='YOUR_CLUSTER_HOST', port=8443, username='YOUR_USERNAME', password='YOUR_CLUSTER_PASSWORD')\n```\n\n### Index data\n\nWe will create an SQL table called `articles` in MyScale to store the embeddings data. The table will include a vector index with a cosine distance metric and a constraint for the length of the embeddings. Use the following code to create and insert data into the articles table:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 089bf370-91cd-4672-aec5-a37690ed7b8b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"089bf370-91cd-4672-aec5-a37690ed7b8b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6dc5fc47-d853-427d-b779-cc8c99434c94\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"d47d5c114761eea9fc5e27ae93bb6ec88b57f8abd19c1a53d16afcd3254b4e6a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9d54e5b4-59f7-4e12-a1b5-fa5879f4b5e8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"61fa91af72ee4f5fd8778ca60338ac801a3bf99a3055b2aa620ede2d6135e6ef\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6833, \"end_char_idx\": 9998, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "## MyScale\nThe next vector database we'll consider is [MyScale](https://myscale.com).\n\n[MyScale](https://myscale.com) is a database built on Clickhouse that combines vector search and SQL analytics to offer a high-performance, streamlined, and fully managed experience. It's designed to facilitate joint queries and analyses on both structured and vector data, with comprehensive SQL support for all data processing.\n\nDeploy and execute vector search with SQL on your cluster within two minutes by using [MyScale Console](https://console.myscale.com).\n\n### Connect to MyScale\n\nFollow the [connections details](https://docs.myscale.com/en/cluster-management/) section to retrieve the cluster host, username, and password information from the MyScale console, and use it to create a connection to your cluster as shown below:\n\n\n```python\n# initialize client\nclient = clickhouse_connect.get_client(host='YOUR_CLUSTER_HOST', port=8443, username='YOUR_USERNAME', password='YOUR_CLUSTER_PASSWORD')\n```\n\n### Index data\n\nWe will create an SQL table called `articles` in MyScale to store the embeddings data. The table will include a vector index with a cosine distance metric and a constraint for the length of the embeddings. Use the following code to create and insert data into the articles table:\n\n\n```python\n# create articles table with vector index\nembedding_len=len(article_df['content_vector'][0]) # 1536\n\nclient.command(f\"\"\"\nCREATE TABLE IF NOT EXISTS default.articles\n(\n    id UInt64,\n    url String,\n    title String,\n    text String,\n    content_vector Array(Float32),\n    CONSTRAINT cons_vector_len CHECK length(content_vector) = {embedding_len},\n    VECTOR INDEX article_content_index content_vector TYPE HNSWFLAT('metric_type=Cosine')\n)\nENGINE = MergeTree ORDER BY id\n\"\"\")\n\n# insert data into the table in batches\nfrom tqdm.auto import tqdm\n\nbatch_size = 100\ntotal_records = len(article_df)\n\n# we only need subset of columns\narticle_df = article_df[['id', 'url', 'title', 'text', 'content_vector']]\n\n# upload data in batches\ndata = article_df.to_records(index=False).tolist()\ncolumn_names = article_df.columns.tolist()\n\nfor i in tqdm(range(0, total_records, batch_size)):\n    i_end = min(i + batch_size, total_records)\n    client.insert(\"default.articles\", data[i:i_end], column_names=column_names)\n```\n\n\n      0%|          | 0/250 [00:00<?, ?it/s]\n\n\nWe need to check the build status of the vector index before proceeding with the search, as it is automatically built in the background.\n\n\n```python\n# check count of inserted data\nprint(f\"articles count: {client.command('SELECT count(*) FROM default.articles')}\")\n\n# check the status of the vector index, make sure vector index is ready with 'Built' status\nget_index_status=\"SELECT status FROM system.vector_indices WHERE name='article_content_index'\"\nprint(f\"index build status: {client.command(get_index_status)}\")\n```\n\n    articles count: 25000\n    index build status: InProgress\n\n\n### Search data\n\nOnce indexed in MyScale, we can perform vector search to find similar content. First, we will use the OpenAI API to generate embeddings for our query. Then, we will perform the vector search using MyScale."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 9d54e5b4-59f7-4e12-a1b5-fa5879f4b5e8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb", "title": "Using MyScale for Embeddings Search", "tokens": 2754, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"9d54e5b4-59f7-4e12-a1b5-fa5879f4b5e8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"367533e5-5816-4237-8c0c-a85a5f65d662\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"fd236f6738bea8e07f4ddca6cb76596d7f2c7a3b7052ead7cb5c93156357c356\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"089bf370-91cd-4672-aec5-a37690ed7b8b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Using_MyScale_for_embeddings_search.ipynb\", \"title\": \"Using MyScale for Embeddings Search\", \"tokens\": 2754, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cfd9467fd965c3588e2a5d9696ed8d394806b2e017ae63e0a64e1470d05d9642\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9151, \"end_char_idx\": 10771, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "ref_doc_id": "367533e5-5816-4237-8c0c-a85a5f65d662", "chroma:document": "0%|          | 0/250 [00:00<?, ?it/s]\n\n\nWe need to check the build status of the vector index before proceeding with the search, as it is automatically built in the background.\n\n\n```python\n# check count of inserted data\nprint(f\"articles count: {client.command('SELECT count(*) FROM default.articles')}\")\n\n# check the status of the vector index, make sure vector index is ready with 'Built' status\nget_index_status=\"SELECT status FROM system.vector_indices WHERE name='article_content_index'\"\nprint(f\"index build status: {client.command(get_index_status)}\")\n```\n\n    articles count: 25000\n    index build status: InProgress\n\n\n### Search data\n\nOnce indexed in MyScale, we can perform vector search to find similar content. First, we will use the OpenAI API to generate embeddings for our query. Then, we will perform the vector search using MyScale.\n\n\n```python\nquery = \"Famous battles in Scottish history\"\n\n# creates embedding vector from user query\nembed = openai.Embedding.create(\n    input=query,\n    model=\"text-embedding-3-small\",\n)[\"data\"][0][\"embedding\"]\n\n# query the database to find the top K similar content to the given query\ntop_k = 10\nresults = client.query(f\"\"\"\nSELECT id, url, title, distance(content_vector, {embed}) as dist\nFROM default.articles\nORDER BY dist\nLIMIT {top_k}\n\"\"\")\n\n# display results\nfor i, r in enumerate(results.named_results()):\n    print(i+1, r['title'])\n```\n\n    1 Battle of Bannockburn\n    2 Wars of Scottish Independence\n    3 1651\n    4 First War of Scottish Independence\n    5 Robert I of Scotland\n    6 841\n    7 1716\n    8 1314\n    9 1263\n    10 William Wallace\n\n\n\n```python\n\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 0efea30f-64f1-495e-8cfe-1dd109fcb240
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb", "title": "Using MyScale as a vector database for OpenAI embeddings", "tokens": 1436, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0efea30f-64f1-495e-8cfe-1dd109fcb240\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b19174-eddb-4a28-8ae6-c321f97ab1d4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b632e784f373b0fcc7ae9a4af37d40088bd2c40a65431319bd4b3f20aa87cfb8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"92bce99f-1698-45ab-abb9-dd69a686c8c8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8a76c3a6e64bd14b5ccee4ff2e05899e0b53e7a20c7c6b09d8dbd1d6b8eec0f0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3102, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "ref_doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "chroma:document": "# Using MyScale as a vector database for OpenAI embeddings\n\nThis notebook provides a step-by-step guide on using MyScale as a vector database for OpenAI embeddings. The process includes:\n\n1. Utilizing precomputed embeddings generated by OpenAI API.\n2. Storing these embeddings in a cloud instance of MyScale.\n3. Converting raw text query to an embedding using OpenAI API.\n4. Leveraging MyScale to perform nearest neighbor search within the created collection.\n\n### What is MyScale\n\n[MyScale](https://myscale.com) is a database built on Clickhouse that combines vector search and SQL analytics to offer a high-performance, streamlined, and fully managed experience. It's designed to facilitate joint queries and analyses on both structured and vector data, with comprehensive SQL support for all data processing.\n\n\n### Deployment options\n\n- Deploy and execute vector search with SQL on your cluster within two minutes by using [MyScale Console](https://console.myscale.com).\n\n\n## Prerequisites\n\nTo follow this guide, you will need to have the following:\n\n1. A MyScale cluster deployed by following the [quickstart guide](https://docs.myscale.com/en/quickstart/).\n2. The 'clickhouse-connect' library to interact with MyScale.\n3. An [OpenAI API key](https://beta.openai.com/account/api-keys) for vectorization of queries.\n\n### Install requirements\n\nThis notebook requires the `openai`, `clickhouse-connect`, as well as some other dependencies. Use the following command to install them:\n\n\n```python\n! pip install openai clickhouse-connect wget pandas\n```\n\n### Prepare your OpenAI API key\n\nTo use the OpenAI API, you'll need to set up an API key. If you don't have one already, you can obtain it from [OpenAI](https://platform.openai.com/account/api-keys).\n\n\n```python\nimport openai\n\n# get API key from on OpenAI website\nopenai.api_key = \"OPENAI_API_KEY\"\n\n# check we have authenticated\nopenai.Engine.list()\n```\n\n## Connect to MyScale\n\nFollow the [connections details](https://docs.myscale.com/en/cluster-management/) section to retrieve the cluster host, username, and password information from the MyScale console, and use it to create a connection to your cluster as shown below:\n\n\n```python\nimport clickhouse_connect\n\n# initialize client\nclient = clickhouse_connect.get_client(host='YOUR_CLUSTER_HOST', port=8443, username='YOUR_USERNAME', password='YOUR_CLUSTER_PASSWORD')\n```\n\n## Load data\n\nWe need to load the dataset of precomputed vector embeddings for Wikipedia articles provided by OpenAI. Use the `wget` package to download the dataset.\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\nAfter the download is complete, extract the file using the `zipfile` package:\n\n\n```python\nimport zipfile\n\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\", \"r\") as zip_ref:\n    zip_ref.extractall(\"../data\")\n```\n\nNow, we can load the data from `vector_database_wikipedia_articles_embedded.csv` into a Pandas DataFrame:"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 92bce99f-1698-45ab-abb9-dd69a686c8c8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb", "title": "Using MyScale as a vector database for OpenAI embeddings", "tokens": 1436, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"92bce99f-1698-45ab-abb9-dd69a686c8c8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b19174-eddb-4a28-8ae6-c321f97ab1d4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b632e784f373b0fcc7ae9a4af37d40088bd2c40a65431319bd4b3f20aa87cfb8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0efea30f-64f1-495e-8cfe-1dd109fcb240\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f33a141bc23a2733bfc6ef917a21145de72bc70eb786f9e0bf3d430e8425c106\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2ab10192-45e2-4acd-8086-3f29e0bb4a85\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6b2f834918e39c5a8b4a72408f93ee070cbc9cf14f0ad2e37a9a14582618fc74\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1486, \"end_char_idx\": 4833, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "ref_doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "chroma:document": "```python\n! pip install openai clickhouse-connect wget pandas\n```\n\n### Prepare your OpenAI API key\n\nTo use the OpenAI API, you'll need to set up an API key. If you don't have one already, you can obtain it from [OpenAI](https://platform.openai.com/account/api-keys).\n\n\n```python\nimport openai\n\n# get API key from on OpenAI website\nopenai.api_key = \"OPENAI_API_KEY\"\n\n# check we have authenticated\nopenai.Engine.list()\n```\n\n## Connect to MyScale\n\nFollow the [connections details](https://docs.myscale.com/en/cluster-management/) section to retrieve the cluster host, username, and password information from the MyScale console, and use it to create a connection to your cluster as shown below:\n\n\n```python\nimport clickhouse_connect\n\n# initialize client\nclient = clickhouse_connect.get_client(host='YOUR_CLUSTER_HOST', port=8443, username='YOUR_USERNAME', password='YOUR_CLUSTER_PASSWORD')\n```\n\n## Load data\n\nWe need to load the dataset of precomputed vector embeddings for Wikipedia articles provided by OpenAI. Use the `wget` package to download the dataset.\n\n\n```python\nimport wget\n\nembeddings_url = \"https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip\"\n\n# The file is ~700 MB so this will take some time\nwget.download(embeddings_url)\n```\n\nAfter the download is complete, extract the file using the `zipfile` package:\n\n\n```python\nimport zipfile\n\nwith zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\", \"r\") as zip_ref:\n    zip_ref.extractall(\"../data\")\n```\n\nNow, we can load the data from `vector_database_wikipedia_articles_embedded.csv` into a Pandas DataFrame:\n\n\n```python\nimport pandas as pd\n\nfrom ast import literal_eval\n\n# read data from csv\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\narticle_df = article_df[['id', 'url', 'title', 'text', 'content_vector']]\n\n# read vectors from strings back into a list\narticle_df[\"content_vector\"] = article_df.content_vector.apply(literal_eval)\narticle_df.head()\n```\n\n## Index data\n\nWe will create an SQL table called `articles` in MyScale to store the embeddings data. The table will include a vector index with a cosine distance metric and a constraint for the length of the embeddings. Use the following code to create and insert data into the articles table:\n\n\n```python\n# create articles table with vector index\nembedding_len=len(article_df['content_vector'][0]) # 1536\n\nclient.command(f\"\"\"\nCREATE TABLE IF NOT EXISTS default.articles\n(\n    id UInt64,\n    url String,\n    title String,\n    text String,\n    content_vector Array(Float32),\n    CONSTRAINT cons_vector_len CHECK length(content_vector) = {embedding_len},\n    VECTOR INDEX article_content_index content_vector TYPE HNSWFLAT('metric_type=Cosine')\n)\nENGINE = MergeTree ORDER BY id\n\"\"\")\n\n# insert data into the table in batches\nfrom tqdm.auto import tqdm\n\nbatch_size = 100\ntotal_records = len(article_df)\n\n# upload data in batches\ndata = article_df.to_records(index=False).tolist()\ncolumn_names = article_df.columns.tolist() \n\nfor i in tqdm(range(0, total_records, batch_size)):\n    i_end = min(i + batch_size, total_records)\n    client.insert(\"default.articles\", data[i:i_end], column_names=column_names)\n```\n\nWe need to check the build status of the vector index before proceeding with the search, as it is automatically built in the background."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 2ab10192-45e2-4acd-8086-3f29e0bb4a85
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb", "title": "Using MyScale as a vector database for OpenAI embeddings", "tokens": 1436, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2ab10192-45e2-4acd-8086-3f29e0bb4a85\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b19174-eddb-4a28-8ae6-c321f97ab1d4\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b632e784f373b0fcc7ae9a4af37d40088bd2c40a65431319bd4b3f20aa87cfb8\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"92bce99f-1698-45ab-abb9-dd69a686c8c8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/myscale/Getting_started_with_MyScale_and_OpenAI.ipynb\", \"title\": \"Using MyScale as a vector database for OpenAI embeddings\", \"tokens\": 1436, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"21498eff6a29b2ceeca6a0f120bb954b1a15c391662db77734e59ce4e4f1239b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3105, \"end_char_idx\": 6286, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "ref_doc_id": "17b19174-eddb-4a28-8ae6-c321f97ab1d4", "chroma:document": "```python\nimport pandas as pd\n\nfrom ast import literal_eval\n\n# read data from csv\narticle_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')\narticle_df = article_df[['id', 'url', 'title', 'text', 'content_vector']]\n\n# read vectors from strings back into a list\narticle_df[\"content_vector\"] = article_df.content_vector.apply(literal_eval)\narticle_df.head()\n```\n\n## Index data\n\nWe will create an SQL table called `articles` in MyScale to store the embeddings data. The table will include a vector index with a cosine distance metric and a constraint for the length of the embeddings. Use the following code to create and insert data into the articles table:\n\n\n```python\n# create articles table with vector index\nembedding_len=len(article_df['content_vector'][0]) # 1536\n\nclient.command(f\"\"\"\nCREATE TABLE IF NOT EXISTS default.articles\n(\n    id UInt64,\n    url String,\n    title String,\n    text String,\n    content_vector Array(Float32),\n    CONSTRAINT cons_vector_len CHECK length(content_vector) = {embedding_len},\n    VECTOR INDEX article_content_index content_vector TYPE HNSWFLAT('metric_type=Cosine')\n)\nENGINE = MergeTree ORDER BY id\n\"\"\")\n\n# insert data into the table in batches\nfrom tqdm.auto import tqdm\n\nbatch_size = 100\ntotal_records = len(article_df)\n\n# upload data in batches\ndata = article_df.to_records(index=False).tolist()\ncolumn_names = article_df.columns.tolist() \n\nfor i in tqdm(range(0, total_records, batch_size)):\n    i_end = min(i + batch_size, total_records)\n    client.insert(\"default.articles\", data[i:i_end], column_names=column_names)\n```\n\nWe need to check the build status of the vector index before proceeding with the search, as it is automatically built in the background.\n\n\n```python\n# check count of inserted data\nprint(f\"articles count: {client.command('SELECT count(*) FROM default.articles')}\")\n\n# check the status of the vector index, make sure vector index is ready with 'Built' status\nget_index_status=\"SELECT status FROM system.vector_indices WHERE name='article_content_index'\"\nprint(f\"index build status: {client.command(get_index_status)}\")\n```\n\n    articles count: 25000\n    index build status: Built\n\n\n## Search data\n\nOnce indexed in MyScale, we can perform vector search to find similar content. First, we will use the OpenAI API to generate embeddings for our query. Then, we will perform the vector search using MyScale.\n\n\n```python\nimport openai\n\nquery = \"Famous battles in Scottish history\"\n\n# creates embedding vector from user query\nembed = openai.Embedding.create(\n    input=query,\n    model=\"text-embedding-3-small\",\n)[\"data\"][0][\"embedding\"]\n\n# query the database to find the top K similar content to the given query\ntop_k = 10\nresults = client.query(f\"\"\"\nSELECT id, url, title, distance(content_vector, {embed}) as dist\nFROM default.articles\nORDER BY dist\nLIMIT {top_k}\n\"\"\")\n\n# display results\nfor i, r in enumerate(results.named_results()):\n    print(i+1, r['title'])\n```\n\n    1 Battle of Bannockburn\n    2 Wars of Scottish Independence\n    3 1651\n    4 First War of Scottish Independence\n    5 Robert I of Scotland\n    6 841\n    7 1716\n    8 1314\n    9 1263\n    10 William Wallace\n\n\n\n```python\n\n```"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: a5133525-325c-413d-bd83-c936f72f877a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb", "title": "Getting Started with Zilliz and OpenAI", "tokens": 5622, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a5133525-325c-413d-bd83-c936f72f877a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9f7c2663-4ef8-4c4d-8683-7e948808037b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b6c5f2c6d1fb2f4a10d0884c8ce3c2f5bff33cf9a04dcde31e91b606e959845a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c2a7e8fb-d2a0-4e8f-aee3-5ff9d12943e1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"94393d94dbf96973f2a08f44bb33ee84468dd8905e7524ff1ae7a5c76f79ab3b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2545, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "ref_doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "chroma:document": "# Getting Started with Zilliz and OpenAI\n### Finding your next book\n\nIn this notebook we will be going over generating embeddings of book descriptions with OpenAI and using those embeddings within Zilliz to find relevant books. The dataset in this example is sourced from HuggingFace datasets, and contains a little over 1 million title-description pairs.\n\nLets begin by first downloading the required libraries for this notebook:\n- `openai` is used for communicating with the OpenAI embedding service\n- `pymilvus` is used for communicating with the Zilliz instance\n- `datasets` is used for downloading the dataset\n- `tqdm` is used for the progress bars\n\n\n\n```python\n! pip install openai pymilvus datasets tqdm\n```\n\n    Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n    Requirement already satisfied: openai in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (0.27.2)\n    Requirement already satisfied: pymilvus in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (2.2.2)\n    Requirement already satisfied: datasets in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (2.10.1)\n    Requirement already satisfied: tqdm in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (4.64.1)\n    Requirement already satisfied: requests>=2.20 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from openai) (2.28.2)\n    Requirement already satisfied: aiohttp in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from openai) (3.8.4)\n    Requirement already satisfied: ujson<=5.4.0,>=2.0.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (5.1.0)\n    Requirement already satisfied: grpcio-tools<=1.48.0,>=1.47.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.47.2)\n    Requirement already satisfied: grpcio<=1.48.0,>=1.47.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.47.2)\n    Requirement already satisfied: mmh3<=3.0.0,>=2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (3.0.0)\n    Requirement already satisfied: pandas>=1.2.4 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.5.3)\n    Requirement already satisfied: numpy>=1.17 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (1.23."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: c2a7e8fb-d2a0-4e8f-aee3-5ff9d12943e1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb", "title": "Getting Started with Zilliz and OpenAI", "tokens": 5622, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c2a7e8fb-d2a0-4e8f-aee3-5ff9d12943e1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9f7c2663-4ef8-4c4d-8683-7e948808037b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b6c5f2c6d1fb2f4a10d0884c8ce3c2f5bff33cf9a04dcde31e91b606e959845a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a5133525-325c-413d-bd83-c936f72f877a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"acf9e7fed1517ea47eb2d8d4e7bbc02e1edc7ec037a8499def13d609157bdffb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4e9152cb-ac1c-4349-950b-dce27a0644ed\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"793c1173d057a697c7b7b16d1877283157031865aa64b7da6ce53cd3487f6780\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1422, \"end_char_idx\": 3680, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "ref_doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "chroma:document": "9/site-packages (from openai) (2.28.2)\n    Requirement already satisfied: aiohttp in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from openai) (3.8.4)\n    Requirement already satisfied: ujson<=5.4.0,>=2.0.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (5.1.0)\n    Requirement already satisfied: grpcio-tools<=1.48.0,>=1.47.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.47.2)\n    Requirement already satisfied: grpcio<=1.48.0,>=1.47.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.47.2)\n    Requirement already satisfied: mmh3<=3.0.0,>=2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (3.0.0)\n    Requirement already satisfied: pandas>=1.2.4 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pymilvus) (1.5.3)\n    Requirement already satisfied: numpy>=1.17 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (1.23.5)\n    Requirement already satisfied: xxhash in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (3.2.0)\n    Requirement already satisfied: responses<0.19 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.18.0)\n    Requirement already satisfied: dill<0.3.7,>=0.3.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.3.6)\n    Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.12.1)\n    Requirement already satisfied: pyarrow>=6.0.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (10.0.1)\n    Requirement already satisfied: multiprocess in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.70.14)\n    Requirement already satisfied: pyyaml>=5.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (5.4.1)\n    Requirement already satisfied: fsspec[http]>=2021.11."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 4e9152cb-ac1c-4349-950b-dce27a0644ed
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb", "title": "Getting Started with Zilliz and OpenAI", "tokens": 5622, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4e9152cb-ac1c-4349-950b-dce27a0644ed\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9f7c2663-4ef8-4c4d-8683-7e948808037b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b6c5f2c6d1fb2f4a10d0884c8ce3c2f5bff33cf9a04dcde31e91b606e959845a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c2a7e8fb-d2a0-4e8f-aee3-5ff9d12943e1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ac7b42538ae6baa8b0a5410a6c47ec16faa159d891e112e7d4abda375e9a29cc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5d8b6a8b-dcd9-41fd-b074-31d73ef57281\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b38132ee4c7934debc6ef3b4d91632fe3f883876a8c25bef4fe996afcfa643c3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2442, \"end_char_idx\": 4804, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "ref_doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "chroma:document": "17 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (1.23.5)\n    Requirement already satisfied: xxhash in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (3.2.0)\n    Requirement already satisfied: responses<0.19 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.18.0)\n    Requirement already satisfied: dill<0.3.7,>=0.3.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.3.6)\n    Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.12.1)\n    Requirement already satisfied: pyarrow>=6.0.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (10.0.1)\n    Requirement already satisfied: multiprocess in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (0.70.14)\n    Requirement already satisfied: pyyaml>=5.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (5.4.1)\n    Requirement already satisfied: fsspec[http]>=2021.11.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (2023.1.0)\n    Requirement already satisfied: packaging in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (23.0)\n    Requirement already satisfied: frozenlist>=1.1.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.3.3)\n    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (4.0.2)\n    Requirement already satisfied: aiosignal>=1.1.2 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.3.1)\n    Requirement already satisfied: attrs>=17.3.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (22.2.0)\n    Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (3.0.1)\n    Requirement already satisfied: yarl<2.0,"}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: 5d8b6a8b-dcd9-41fd-b074-31d73ef57281
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb", "title": "Getting Started with Zilliz and OpenAI", "tokens": 5622, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5d8b6a8b-dcd9-41fd-b074-31d73ef57281\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9f7c2663-4ef8-4c4d-8683-7e948808037b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b6c5f2c6d1fb2f4a10d0884c8ce3c2f5bff33cf9a04dcde31e91b606e959845a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4e9152cb-ac1c-4349-950b-dce27a0644ed\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a57ab2c1e9cb75fd97f5f25b6f71b7585a6a65c5182a99479d5909c2d254406e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c9f033a8-7cd9-49fd-87f5-84d7c608e557\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"86ee24fa828472193a329b5d1293c42eed72a0982f37a71454a210f8f4c3a8cb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3583, \"end_char_idx\": 5805, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "ref_doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "chroma:document": "9/site-packages (from datasets) (5.4.1)\n    Requirement already satisfied: fsspec[http]>=2021.11.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (2023.1.0)\n    Requirement already satisfied: packaging in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from datasets) (23.0)\n    Requirement already satisfied: frozenlist>=1.1.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.3.3)\n    Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (4.0.2)\n    Requirement already satisfied: aiosignal>=1.1.2 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.3.1)\n    Requirement already satisfied: attrs>=17.3.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (22.2.0)\n    Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (3.0.1)\n    Requirement already satisfied: yarl<2.0,>=1.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.8.2)\n    Requirement already satisfied: multidict<7.0,>=4.5 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (6.0.4)\n    Requirement already satisfied: six>=1.5.2 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio<=1.48.0,>=1.47.0->pymilvus) (1.16.0)\n    Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio-tools<=1.48.0,>=1.47.0->pymilvus) (3.20.1)\n    Requirement already satisfied: setuptools in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio-tools<=1.48.0,>=1.47.0->pymilvus) (65.6.3)\n    Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.2."}


created_at: 2024-07-28 18:59:18
topic: persistent://default/default/4bfc3c11-fb71-4cc9-8c12-01ed076a0e01
id: c9f033a8-7cd9-49fd-87f5-84d7c608e557
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb", "title": "Getting Started with Zilliz and OpenAI", "tokens": 5622, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c9f033a8-7cd9-49fd-87f5-84d7c608e557\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"9f7c2663-4ef8-4c4d-8683-7e948808037b\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b6c5f2c6d1fb2f4a10d0884c8ce3c2f5bff33cf9a04dcde31e91b606e959845a\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5d8b6a8b-dcd9-41fd-b074-31d73ef57281\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/vector_databases/zilliz/Getting_started_with_Zilliz_and_OpenAI.ipynb\", \"title\": \"Getting Started with Zilliz and OpenAI\", \"tokens\": 5622, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"5f71d277c5719a9d2b93766401ca64f86f9aa3bc64bf00f9e06e29288437a415\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"da2b8a8f-e54b-4bac-8192-2b77e7cf1dea\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"57939f9c9a9f8dba31243cd8861a5cc10c2daee5cb239cf4c2188d4daf93ab76\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4713, \"end_char_idx\": 6854, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "ref_doc_id": "9f7c2663-4ef8-4c4d-8683-7e948808037b", "chroma:document": "9/site-packages (from aiohttp->openai) (3.0.1)\n    Requirement already satisfied: yarl<2.0,>=1.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (1.8.2)\n    Requirement already satisfied: multidict<7.0,>=4.5 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from aiohttp->openai) (6.0.4)\n    Requirement already satisfied: six>=1.5.2 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio<=1.48.0,>=1.47.0->pymilvus) (1.16.0)\n    Requirement already satisfied: protobuf<4.0dev,>=3.12.0 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio-tools<=1.48.0,>=1.47.0->pymilvus) (3.20.1)\n    Requirement already satisfied: setuptools in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from grpcio-tools<=1.48.0,>=1.47.0->pymilvus) (65.6.3)\n    Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.5.0)\n    Requirement already satisfied: filelock in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.9.0)\n    Requirement already satisfied: python-dateutil>=2.8.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pandas>=1.2.4->pymilvus) (2.8.2)\n    Requirement already satisfied: pytz>=2020.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from pandas>=1.2.4->pymilvus) (2022.7.1)\n    Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from requests>=2.20->openai) (1.26.14)\n    Requirement already satisfied: idna<4,>=2.5 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from requests>=2.20->openai) (3.4)\n    Requirement already satisfied: certifi>=2017.4.17 in /Users/filiphaltmayer/miniconda3/envs/haystack/lib/python3.9/site-packages (from requests>=2.20->openai) (2022.12."}


