# embeddings_queue

created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 5255a6a8-54d9-4d12-8353-83905e991e98
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5255a6a8-54d9-4d12-8353-83905e991e98\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"25f7ab39-e8d9-4d63-803e-bc8edcef554a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f64af391e3d219d48b9c0d4187474c00ad6ac7c4f830468b5896d5fec20d2b0d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2377472f-c78d-4b1d-ae74-d24a5a819fd2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7cd47454946ab6139afae71ab201585ccc7d5c8834aa578c5fc36a089c5db99f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5078, \"end_char_idx\": 7275, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "```python\nimport random\n\ndef get_random_similar_contexts(question, context, file_id=olympics_search_fileid, search_model='ada', max_rerank=10):\n    \"\"\"\n    Find similar contexts to the given context using the search file\n    \"\"\"\n    try:\n        # TODO: openai.Engine(search_model) is deprecated\n        results = openai.Engine(search_model).search(\n            search_model=search_model, \n            query=question, \n            max_rerank=max_rerank,\n            file=file_id\n        )\n        candidates = []\n        for result in results['data'][:3]:\n            if result['text'] == context:\n                continue\n            candidates.append(result['text'])\n        random_candidate = random.choice(candidates)\n        return random_candidate\n    except Exception as e:\n        print(e)\n        return \"\"\n\ndef create_fine_tuning_dataset(df, discriminator=False, n_negative=1, add_related=False):\n    \"\"\"\n    Create a dataset for fine tuning the OpenAI model; either for a discriminator model, \n    or a model specializing in Q&A, where it says if no relevant context is found.\n\n    Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the question, answer and context pairs\n    discriminator: bool\n        Whether to create a dataset for the discriminator\n    n_negative: int\n        The number of random negative samples to add (using a random context)\n    add_related: bool\n        Whether to add the related contexts to the correct context. These are hard negative examples\n\n    Returns\n    -------\n    pd.DataFrame\n        The dataframe containing the prompts and completions, ready for fine-tuning\n    \"\"\"\n    rows = []\n    for i, row in df.iterrows():\n        for q, a in zip((\"1.\" + row.questions).split('\\n'), (\"1.\" + row.answers).split('\\n')):\n            if len(q) >10 and len(a) >10:\n                if discriminator:\n                    rows.append({\"prompt\":f\"{row.context}\\nQuestion: {q[2:].strip()}\\n Related:\", \"completion\":f\" yes\"})\n                else:\n                    rows.append({\"prompt\":f\"{row.context}\\nQuestion: {q[2:].strip()}\\nAnswer:\", \"completion\":f\" {a[2:].strip()}\"})\n\n    for i, row in df.iterrows():\n        for q in (\"1.\""}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2377472f-c78d-4b1d-ae74-d24a5a819fd2
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2377472f-c78d-4b1d-ae74-d24a5a819fd2\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5255a6a8-54d9-4d12-8353-83905e991e98\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"0e7d6e4cd48b2b65afb3b048a7eb02e82e76c4ce9bcc3de23db8cc7f488f17c5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ce6bac88-e4aa-4fa6-bc06-32b0203da502\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4952c8c96d01674ece45e094eebc13b338fd2ce070d706a4f9c1675cac1e3b7b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6171, \"end_char_idx\": 9452, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "Parameters\n    ----------\n    df: pd.DataFrame\n        The dataframe containing the question, answer and context pairs\n    discriminator: bool\n        Whether to create a dataset for the discriminator\n    n_negative: int\n        The number of random negative samples to add (using a random context)\n    add_related: bool\n        Whether to add the related contexts to the correct context. These are hard negative examples\n\n    Returns\n    -------\n    pd.DataFrame\n        The dataframe containing the prompts and completions, ready for fine-tuning\n    \"\"\"\n    rows = []\n    for i, row in df.iterrows():\n        for q, a in zip((\"1.\" + row.questions).split('\\n'), (\"1.\" + row.answers).split('\\n')):\n            if len(q) >10 and len(a) >10:\n                if discriminator:\n                    rows.append({\"prompt\":f\"{row.context}\\nQuestion: {q[2:].strip()}\\n Related:\", \"completion\":f\" yes\"})\n                else:\n                    rows.append({\"prompt\":f\"{row.context}\\nQuestion: {q[2:].strip()}\\nAnswer:\", \"completion\":f\" {a[2:].strip()}\"})\n\n    for i, row in df.iterrows():\n        for q in (\"1.\" + row.questions).split('\\n'):\n            if len(q) >10:\n                for j in range(n_negative + (2 if add_related else 0)):\n                    random_context = \"\"\n                    if j == 0 and add_related:\n                        # add the related contexts based on originating from the same wikipedia page\n                        subset = df[(df.title == row.title) & (df.context != row.context)]\n                        \n                        if len(subset) < 1:\n                            continue\n                        random_context = subset.sample(1).iloc[0].context\n                    if j == 1 and add_related:\n                        # add the related contexts based on the most similar contexts according to the search\n                        random_context = get_random_similar_contexts(q[2:].strip(), row.context, search_model='ada', max_rerank=10)\n                    else:\n                        while True:\n                            # add random context, which isn't the correct context\n                            random_context = df.sample(1).iloc[0].context\n                            if random_context != row.context:\n                                break\n                    if discriminator:\n                        rows.append({\"prompt\":f\"{random_context}\\nQuestion: {q[2:].strip()}\\n Related:\", \"completion\":f\" no\"})\n                    else:\n                        rows.append({\"prompt\":f\"{random_context}\\nQuestion: {q[2:].strip()}\\nAnswer:\", \"completion\":f\" No appropriate context found to answer the question.\"})\n\n    return pd.DataFrame(rows) \n```\n\nWe apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.\n\n\n```python\nfor name, is_disc in [('discriminator', True), ('qa', False)]:\n    for train_test, dt in [('train', train_df), ('test', test_df)]:\n        ft = create_fine_tuning_dataset(dt, discriminator=is_disc, n_negative=1, add_related=True)\n        ft.to_json(f'{name}_{train_test}.jsonl', orient='records', lines=True)\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ce6bac88-e4aa-4fa6-bc06-32b0203da502
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ce6bac88-e4aa-4fa6-bc06-32b0203da502\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2377472f-c78d-4b1d-ae74-d24a5a819fd2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"2824c393eb81a87f87cd12801cb9e646b8f12657d1ffb3d5e3aa3873af1b7418\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"20964ebd-0866-4604-be6f-4f1a5d8eb4e4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"480a487e2559ab1c532fa75f9b253037d81cee8e2b0cdc80cebb6e7fc7199876\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8842, \"end_char_idx\": 11388, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "return pd.DataFrame(rows) \n```\n\nWe apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.\n\n\n```python\nfor name, is_disc in [('discriminator', True), ('qa', False)]:\n    for train_test, dt in [('train', train_df), ('test', test_df)]:\n        ft = create_fine_tuning_dataset(dt, discriminator=is_disc, n_negative=1, add_related=True)\n        ft.to_json(f'{name}_{train_test}.jsonl', orient='records', lines=True)\n```\n\n\n\n\n    \n\n\n\nWe formatted the data according to the recommendations from the fine-tuning tool, which is available using\n> openai tools fine_tunes.prepare_data -f qa_train.jsonl\n\nWe highly recommend that you use this tool, which suggests improvements in your data formatting for fine-tuning.\n\n\n## 3.2 Submit the datasets for fine-tuning\n\n\n```python\n!openai api fine_tunes.create -t \"olympics-data/discriminator_train.jsonl\" -v \"olympics-data/discriminator_test.jsonl\" --batch_size 16  --compute_classification_metrics --classification_positive_class \" yes\" --model ada\n```\n\n\n\n\n    \n\n\n\n\n```python\n!openai api fine_tunes.create -t \"olympics-data/qa_train.jsonl\" -v \"olympics-data/qa_test.jsonl\" --batch_size 16\n```\n\n\n\n\n    \n\n\n\n## 3.3 Using the fine-tuned models\n\nWe will now use the fine-tuned discriminator and the fine-tuned Q&A model. By requesting logprobs, we can see how certain the discriminator is in a `yes` vs `no` answer.\n\n\n```python\nft_discriminator = \"curie:ft-openai-internal-2021-08-23-23-58-57\"\nft_qa = \"curie:ft-openai-internal-2021-08-23-17-54-10\"\n\ndef apply_ft_discriminator(context, question, discriminator_model):\n    \"\"\"\n    Apply the fine tuned discriminator to a question, to assess whether it can be answered from the context.\n    \"\"\"\n    prompt = f\"{context}\\nQuestion: {question}\\n Related:\"\n    result = openai.chat.completions.create(model=discriminator_model, prompt=prompt, max_tokens=1, temperature=0, top_p=1, n=1, logprobs=2)\n    return result['choices'][0]['logprobs']['top_logprobs']\n\napply_ft_discriminator('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', \n                        'What was the first human-made object in space?', ft_discriminator)\n```\n\n\n\n\n    [<OpenAIObject at 0x7fe812e602b0> JSON: {\n       \" no\": -10.819577,\n       \" yes\": -2.045765e-05\n     }]\n\n\n\nWe can see that the model can generalize well to different contexts and questions."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 20964ebd-0866-4604-be6f-4f1a5d8eb4e4
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"20964ebd-0866-4604-be6f-4f1a5d8eb4e4\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ce6bac88-e4aa-4fa6-bc06-32b0203da502\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"bb266521f3e9c76b049097d1880ecc80308124e96d0c5cd75fc2e0b3ee03bda2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"09034718-6b4e-4a85-825c-1f775d8786e8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"27ff64ed005db3e16c69aebcedb676879b47791f883923d5c60482304732d79e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10037, \"end_char_idx\": 12758, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "```python\n!openai api fine_tunes.create -t \"olympics-data/qa_train.jsonl\" -v \"olympics-data/qa_test.jsonl\" --batch_size 16\n```\n\n\n\n\n    \n\n\n\n## 3.3 Using the fine-tuned models\n\nWe will now use the fine-tuned discriminator and the fine-tuned Q&A model. By requesting logprobs, we can see how certain the discriminator is in a `yes` vs `no` answer.\n\n\n```python\nft_discriminator = \"curie:ft-openai-internal-2021-08-23-23-58-57\"\nft_qa = \"curie:ft-openai-internal-2021-08-23-17-54-10\"\n\ndef apply_ft_discriminator(context, question, discriminator_model):\n    \"\"\"\n    Apply the fine tuned discriminator to a question, to assess whether it can be answered from the context.\n    \"\"\"\n    prompt = f\"{context}\\nQuestion: {question}\\n Related:\"\n    result = openai.chat.completions.create(model=discriminator_model, prompt=prompt, max_tokens=1, temperature=0, top_p=1, n=1, logprobs=2)\n    return result['choices'][0]['logprobs']['top_logprobs']\n\napply_ft_discriminator('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', \n                        'What was the first human-made object in space?', ft_discriminator)\n```\n\n\n\n\n    [<OpenAIObject at 0x7fe812e602b0> JSON: {\n       \" no\": -10.819577,\n       \" yes\": -2.045765e-05\n     }]\n\n\n\nWe can see that the model can generalize well to different contexts and questions. \n\n\n```python\ndef apply_ft_qa_answer(context, question, answering_model):\n    \"\"\"\n    Apply the fine tuned discriminator to a question\n    \"\"\"\n    prompt = f\"{context}\\nQuestion: {question}\\nAnswer:\"\n    result = openai.chat.completions.create(model=answering_model, prompt=prompt, max_tokens=30, temperature=0, top_p=1, n=1, stop=['.','\\n'])\n    return result['choices'][0]['text']\n\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', \n                    'What was the first human-made object in space?', ft_qa)\n\n```\n\n\n\n\n    ' The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957'\n\n\n\nWe can see that the model can answer the question, when the context is appropriate.\n\n\n```python\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',\n                    'What is impressive about the Soviet Union?', ft_qa)\n```\n\n\n\n\n    ' The Soviet Union was the first country to successfully launch a satellite into space'\n\n\n\n\n```python\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',\n                    'How many cars were produced in the Soviet Union in 1970?', ft_qa)\n```\n\n\n\n\n    ' No appropriate context found to answer the question'"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 09034718-6b4e-4a85-825c-1f775d8786e8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"09034718-6b4e-4a85-825c-1f775d8786e8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"20964ebd-0866-4604-be6f-4f1a5d8eb4e4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"86cba393c0e54b27fcf9e8d6542829db024d4038284898bd3ba283621ca2c53e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"99a1dc68-13ad-4fa5-8f2c-10aa4ec8b2e7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dd0e2870aa9a41a060aa057792de758d51ceea743c78493f1bdf148ccfa7b3ba\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11198, \"end_char_idx\": 14490, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "[<OpenAIObject at 0x7fe812e602b0> JSON: {\n       \" no\": -10.819577,\n       \" yes\": -2.045765e-05\n     }]\n\n\n\nWe can see that the model can generalize well to different contexts and questions. \n\n\n```python\ndef apply_ft_qa_answer(context, question, answering_model):\n    \"\"\"\n    Apply the fine tuned discriminator to a question\n    \"\"\"\n    prompt = f\"{context}\\nQuestion: {question}\\nAnswer:\"\n    result = openai.chat.completions.create(model=answering_model, prompt=prompt, max_tokens=30, temperature=0, top_p=1, n=1, stop=['.','\\n'])\n    return result['choices'][0]['text']\n\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', \n                    'What was the first human-made object in space?', ft_qa)\n\n```\n\n\n\n\n    ' The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957'\n\n\n\nWe can see that the model can answer the question, when the context is appropriate.\n\n\n```python\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',\n                    'What is impressive about the Soviet Union?', ft_qa)\n```\n\n\n\n\n    ' The Soviet Union was the first country to successfully launch a satellite into space'\n\n\n\n\n```python\napply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',\n                    'How many cars were produced in the Soviet Union in 1970?', ft_qa)\n```\n\n\n\n\n    ' No appropriate context found to answer the question'\n\n\n\nWe can see that the model knows when to answer the question, and when to say that insufficient context is present to answer the question.\n\nWe can also combine a discriminator and a base model, or a fine-tuned Q&A model. Discriminator can essentially serve as a decision whether the question can be answered given the context or not.\n\n\n```python\ndef answer_question_conditionally(answering_model, discriminator_model, context, question, discriminator_logprob_yes_modifier=0):\n    logprobs = apply_ft_discriminator(context, question, discriminator_model)\n    yes_logprob = logprobs[' yes'] if ' yes' in logprobs else -100\n    no_logprob = logprobs[' no'] if ' no' in logprobs else -100\n    if yes_logprob + discriminator_logprob_yes_modifier < no_logprob:\n        return \" No appropriate context found to answer the question based on the discriminator.\"\n    return apply_ft_qa_answer(context, question, answering_model)\nanswer_question_conditionally(ft_qa, ft_discriminator, \n                                \"Crowdless games are a rare although not unheard-of occurrence in sports. \\\n                                 When they do occur, it is usually the result of events beyond the control \\\n                                 of the teams or fans, such as weather-related concerns, public health concerns, \\\n                                 or wider civil disturbances unrelated to the game. For instance, \\\n                                 the COVID-19 pandemic caused many sports leagues around the world \\\n                                 to be played behind closed doors.\",\n                                \"Could weather cause a sport event to have no crowd?\")\n```\n\n\n\n\n    ' Weather could cause a sport event to have no crowd'"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 99a1dc68-13ad-4fa5-8f2c-10aa4ec8b2e7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"99a1dc68-13ad-4fa5-8f2c-10aa4ec8b2e7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"d678dd15-f627-407c-881b-3f384b1d9028\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"09034718-6b4e-4a85-825c-1f775d8786e8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb\", \"title\": \"3. Train a fine-tuning model specialized for Q&A\", \"tokens\": 3706, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"c21064ea8f2663339fb675bcc43976902e954243f7556353a11c1e4c35487814\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12704, \"end_char_idx\": 15566, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "d678dd15-f627-407c-881b-3f384b1d9028", "doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "ref_doc_id": "d678dd15-f627-407c-881b-3f384b1d9028", "chroma:document": "' No appropriate context found to answer the question'\n\n\n\nWe can see that the model knows when to answer the question, and when to say that insufficient context is present to answer the question.\n\nWe can also combine a discriminator and a base model, or a fine-tuned Q&A model. Discriminator can essentially serve as a decision whether the question can be answered given the context or not.\n\n\n```python\ndef answer_question_conditionally(answering_model, discriminator_model, context, question, discriminator_logprob_yes_modifier=0):\n    logprobs = apply_ft_discriminator(context, question, discriminator_model)\n    yes_logprob = logprobs[' yes'] if ' yes' in logprobs else -100\n    no_logprob = logprobs[' no'] if ' no' in logprobs else -100\n    if yes_logprob + discriminator_logprob_yes_modifier < no_logprob:\n        return \" No appropriate context found to answer the question based on the discriminator.\"\n    return apply_ft_qa_answer(context, question, answering_model)\nanswer_question_conditionally(ft_qa, ft_discriminator, \n                                \"Crowdless games are a rare although not unheard-of occurrence in sports. \\\n                                 When they do occur, it is usually the result of events beyond the control \\\n                                 of the teams or fans, such as weather-related concerns, public health concerns, \\\n                                 or wider civil disturbances unrelated to the game. For instance, \\\n                                 the COVID-19 pandemic caused many sports leagues around the world \\\n                                 to be played behind closed doors.\",\n                                \"Could weather cause a sport event to have no crowd?\")\n```\n\n\n\n\n    ' Weather could cause a sport event to have no crowd'\n\n\n\nThe above function illustrates how to potentially combine a discriminator and a fine-tuned Q&A model. This gives a more fine-grained control over how certain we want the model to be before it answers the question.\n\nWe'll now take a look on how answers endpoint works - combining search to retrieve the relevant context from a knowledge base, and then using the fine-tuned Q&A model to answer the question.\n\n## 3.4 Answering the question based on a knowledge base\nFinally we can use a logic similar to the [/answers](https://beta.openai.com/docs/api-reference/answers) endpoint, where we first search for the relevant context, and then ask a Q&A model to answer the question given that context. If you'd like to see the implementation details, check out the [`answers_with_ft.py`](answers_with_ft.py) file.\n\n\n```python\nfrom answers_with_ft import answer_question\nanswer_question(olympics_search_fileid, ft_qa, \"Which country won the Women's football tournament at the 2020 Olympic games?\")\n```\n\n\n\n\n    \" Canada won the Women's football tournament at the 2020 Olympic games\""}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 5b921ab4-1eed-4845-a299-8c62cfb6a722
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5b921ab4-1eed-4845-a299-8c62cfb6a722\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7ea9ff6c1202762e7155acb3717eb1a0e74993130131a336b46f05cfdab5f1d4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"926a88a4-0470-4a11-972d-7195159fe331\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"abf35b6ef3cb253f1c316c5be58c6c77bc7ed80f0ffa7eec342ad985636ebf89\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2536, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "ref_doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "chroma:document": "<span style=\"color:orange; font-weight:bold\">Note: To answer questions based on text documents, we recommend the procedure in <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering using Embeddings</a>. Some of the code below may rely on <a href=\"https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints\">deprecated API endpoints</a>.</span>\n\n# 1. Collect Wikipedia data about Olympic Games 2020\n\nThe idea of this project is to create a question answering model, based on a few paragraphs of provided text. Base GPT-3 models do a good job at answering questions when the answer is contained within the paragraph, however if the answer isn't contained, the base models tend to try their best to answer anyway, often leading to confabulated answers. \n\nTo create a model which answers questions only if there is sufficient context for doing so, we first create a dataset of questions and answers based on paragraphs of text. In order to train the model to answer only when the answer is present, we also add adversarial examples, where the question doesn't match the context. In those cases, we ask the model to output \"No sufficient context for answering the question\". \n\nWe will perform this task in three notebooks:\n1. The first (this) notebook focuses on collecting recent data, which GPT-3 didn't see during its pre-training. We picked the topic of Olympic Games 2020 (which actually took place in the summer of 2021), and downloaded 713 unique pages. We organized the dataset by individual sections, which will serve as context for asking and answering the questions.\n2. The [second notebook](olympics-2-create-qa.ipynb) will utilize Davinci-instruct to ask a few questions based on a Wikipedia section, as well as answer those questions, based on that section.\n3. The [third notebook](olympics-3-train-qa.ipynb) will utilize the dataset of context, question and answer pairs to additionally create adversarial questions and context pairs, where the question was not generated on that context. In those cases the model will be prompted to answer \"No sufficient context for answering the question\". We will also train a discriminator model, which predicts whether the question can be answered based on the context or not.\n\n## 1.1 Data extraction using the wikipedia API\nExtracting the data will take about half an hour, and processing will likely take about as much.\n\n\n```python\nimport pandas as pd\nimport wikipedia"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 926a88a4-0470-4a11-972d-7195159fe331
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"926a88a4-0470-4a11-972d-7195159fe331\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7ea9ff6c1202762e7155acb3717eb1a0e74993130131a336b46f05cfdab5f1d4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5b921ab4-1eed-4845-a299-8c62cfb6a722\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"348abafd00891507cdcf2bc20c2701e3b9f796f9bb9538b7af4aced95fccc920\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"dc514b42-8ce8-4f5a-bfcf-6751c7afd121\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8a3a087c022e785b48b6f357cb462adc284d1c5d1116f1be227075e044f7641a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2490, \"end_char_idx\": 5382, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "ref_doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "chroma:document": "```python\nimport pandas as pd\nimport wikipedia\n\n\ndef filter_olympic_2020_titles(titles):\n    \"\"\"\n    Get the titles which are related to Olympic games hosted in 2020, given a list of titles\n    \"\"\"\n    titles = [title for title in titles if '2020' in title and 'olympi' in title.lower()]\n    \n    return titles\n\ndef get_wiki_page(title):\n    \"\"\"\n    Get the wikipedia page given a title\n    \"\"\"\n    try:\n        return wikipedia.page(title)\n    except wikipedia.exceptions.DisambiguationError as e:\n        return wikipedia.page(e.options[0])\n    except wikipedia.exceptions.PageError as e:\n        return None\n\ndef recursively_find_all_pages(titles, titles_so_far=set()):\n    \"\"\"\n    Recursively find all the pages that are linked to the Wikipedia titles in the list\n    \"\"\"\n    all_pages = []\n    \n    titles = list(set(titles) - titles_so_far)\n    titles = filter_olympic_2020_titles(titles)\n    titles_so_far.update(titles)\n    for title in titles:\n        page = get_wiki_page(title)\n        if page is None:\n            continue\n        all_pages.append(page)\n\n        new_pages = recursively_find_all_pages(page.links, titles_so_far)\n        for pg in new_pages:\n            if pg.title not in [p.title for p in all_pages]:\n                all_pages.append(pg)\n        titles_so_far.update(page.links)\n    return all_pages\n\n\npages = recursively_find_all_pages([\"2020 Summer Olympics\"])\nlen(pages)\n```\n\n\n\n\n    909\n\n\n\n## 1.2 Filtering the Wikipedia pages and splitting them into sections by headings\nWe remove sections unlikely to contain textual information, and ensure that each section is not longer than the token limit\n\n\n```python\n\nimport re\nfrom typing import Set\nfrom transformers import GPT2TokenizerFast\n\nimport numpy as np\nfrom nltk.tokenize import sent_tokenize\n\ntokenizer = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n\ndef count_tokens(text: str) -> int:\n    \"\"\"count the number of tokens in a string\"\"\"\n    return len(tokenizer.encode(text))\n\ndef reduce_long(\n    long_text: str, long_text_tokens: bool = False, max_len: int = 590\n) -> str:\n    \"\"\"\n    Reduce a long text to a maximum of `max_len` tokens by potentially cutting at a sentence end\n    \"\"\"\n    if not long_text_tokens:\n        long_text_tokens = count_tokens(long_text)\n    if long_text_tokens > max_len:\n        sentences = sent_tokenize(long_text.replace(\"\\n\", \" \"))\n        ntokens = 0\n        for i, sentence in enumerate(sentences):\n            ntokens += 1 + count_tokens(sentence)\n            if ntokens > max_len:\n                return \". \".join(sentences[:i]) + \".\"\n\n    return long_text\n\ndiscard_categories = ['See also', 'References', 'External links', 'Further reading', \"Footnotes\",\n    \"Bibliography\", \"Sources\", \"Citations\", \"Literature\", \"Footnotes\", \"Notes and references\",\n    \"Photo gallery\", \"Works cited\", \"Photos\", \"Gallery\", \"Notes\", \"References and sources\",\n    \"References and notes\",]"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: dc514b42-8ce8-4f5a-bfcf-6751c7afd121
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"dc514b42-8ce8-4f5a-bfcf-6751c7afd121\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7ea9ff6c1202762e7155acb3717eb1a0e74993130131a336b46f05cfdab5f1d4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"926a88a4-0470-4a11-972d-7195159fe331\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"48394ece7edf83343860ae3d4d0418240d4164a253ba6c2cc96b70fb27425bf7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a6bf6803-4365-42cd-aa56-9ea0c7e0beea\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"56fa6a030c3d7a503e8064cea6a1b7c497fb0554dca98016efbe46ad3037f503\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3822, \"end_char_idx\": 7820, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "ref_doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "chroma:document": "pages = recursively_find_all_pages([\"2020 Summer Olympics\"])\nlen(pages)\n```\n\n\n\n\n    909\n\n\n\n## 1.2 Filtering the Wikipedia pages and splitting them into sections by headings\nWe remove sections unlikely to contain textual information, and ensure that each section is not longer than the token limit\n\n\n```python\n\nimport re\nfrom typing import Set\nfrom transformers import GPT2TokenizerFast\n\nimport numpy as np\nfrom nltk.tokenize import sent_tokenize\n\ntokenizer = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n\ndef count_tokens(text: str) -> int:\n    \"\"\"count the number of tokens in a string\"\"\"\n    return len(tokenizer.encode(text))\n\ndef reduce_long(\n    long_text: str, long_text_tokens: bool = False, max_len: int = 590\n) -> str:\n    \"\"\"\n    Reduce a long text to a maximum of `max_len` tokens by potentially cutting at a sentence end\n    \"\"\"\n    if not long_text_tokens:\n        long_text_tokens = count_tokens(long_text)\n    if long_text_tokens > max_len:\n        sentences = sent_tokenize(long_text.replace(\"\\n\", \" \"))\n        ntokens = 0\n        for i, sentence in enumerate(sentences):\n            ntokens += 1 + count_tokens(sentence)\n            if ntokens > max_len:\n                return \". \".join(sentences[:i]) + \".\"\n\n    return long_text\n\ndiscard_categories = ['See also', 'References', 'External links', 'Further reading', \"Footnotes\",\n    \"Bibliography\", \"Sources\", \"Citations\", \"Literature\", \"Footnotes\", \"Notes and references\",\n    \"Photo gallery\", \"Works cited\", \"Photos\", \"Gallery\", \"Notes\", \"References and sources\",\n    \"References and notes\",]\n\n\ndef extract_sections(\n    wiki_text: str,\n    title: str,\n    max_len: int = 1500,\n    discard_categories: Set[str] = discard_categories,\n) -> str:\n    \"\"\"\n    Extract the sections of a Wikipedia page, discarding the references and other low information sections\n    \"\"\"\n    if len(wiki_text) == 0:\n        return []\n\n    # find all headings and the corresponding contents\n    headings = re.findall(\"==+ .* ==+\", wiki_text)\n    for heading in headings:\n        wiki_text = wiki_text.replace(heading, \"==+ !! ==+\")\n    contents = wiki_text.split(\"==+ !! ==+\")\n    contents = [c.strip() for c in contents]\n    assert len(headings) == len(contents) - 1\n\n    cont = contents.pop(0).strip()\n    outputs = [(title, \"Summary\", cont, count_tokens(cont)+4)]\n    \n    # discard the discard categories, accounting for a tree structure\n    max_level = 100\n    keep_group_level = max_level\n    remove_group_level = max_level\n    nheadings, ncontents = [], []\n    for heading, content in zip(headings, contents):\n        plain_heading = \" \".join(heading.split(\" \")[1:-1])\n        num_equals = len(heading.split(\" \")[0])\n        if num_equals <= keep_group_level:\n            keep_group_level = max_level\n\n        if num_equals > remove_group_level:\n            if (\n                num_equals <= keep_group_level\n            ):\n                continue\n        keep_group_level = max_level\n        if plain_heading in discard_categories:\n            remove_group_level = num_equals\n            keep_group_level = max_level\n            continue\n        nheadings.append(heading.replace(\"=\", \"\").strip())\n        ncontents.append(content)\n        remove_group_level = max_level\n\n    # count the tokens of each section\n    ncontent_ntokens = [\n        count_tokens(c)\n        + 3\n        + count_tokens(\" \".join(h.split(\" \")[1:-1]))\n        - (1 if len(c) == 0 else 0)\n        for h, c in zip(nheadings, ncontents)\n    ]\n\n    # Create a tuple of (title, section_name, content, number of tokens)\n    outputs += [(title, h, c, t) if t<max_len \n                else (title, h, reduce_long(c, max_len), count_tokens(reduce_long(c,max_len))) \n                    for h, c, t in zip(nheadings, ncontents, ncontent_ntokens)]\n    \n    return outputs\n\n# Example page being processed into sections\nbermuda_page = get_wiki_page('Bermuda at the 2020 Summer Olympics')\nber = extract_sections(bermuda_page.content, bermuda_page.title)\n\n# Example section\nber[-1]\n\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a6bf6803-4365-42cd-aa56-9ea0c7e0beea
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a6bf6803-4365-42cd-aa56-9ea0c7e0beea\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7ea9ff6c1202762e7155acb3717eb1a0e74993130131a336b46f05cfdab5f1d4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"dc514b42-8ce8-4f5a-bfcf-6751c7afd121\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"4d1726e73d28316c3f7bc411ddfac2bd6a705d6eaafcc1b3b525a053f587a81f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"321c2a15-b8ad-4113-8300-e7d921a97d94\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"508edbe293cf0bbc794a55ce9d45113eb806f2234d4c3bc76f9449cff669115e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7829, \"end_char_idx\": 10513, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "ref_doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "chroma:document": "('Bermuda at the 2020 Summer Olympics',\n     'Equestrian',\n     \"Bermuda entered one dressage rider into the Olympic competition by finishing in the top four, outside the group selection, of the individual FEI Olympic Rankings for Groups D and E (North, Central, and South America), marking the country's recurrence to the sport after an eight-year absence. The quota was later withdrawn, following an injury of Annabelle Collins' main horse Joyero and a failure to obtain minimum eligibility requirements (MER) aboard a new horse Chuppy Checker.\",\n     104)\n\n\n\n### 1.2.1 We create a dataset and filter out any sections with fewer than 40 tokens, as those are unlikely to contain enough context to ask a good question.\n\n\n```python\nres = []\nfor page in pages:\n    res += extract_sections(page.content, page.title)\ndf = pd.DataFrame(res, columns=[\"title\", \"heading\", \"content\", \"tokens\"])\ndf = df[df.tokens>40]\ndf = df.drop_duplicates(['title','heading'])\ndf = df.reset_index().drop('index',axis=1) # reset index\ndf.head()\n```\n\n    Token indices sequence length is longer than the specified maximum sequence length for this model (1060 > 1024). Running this sequence through the model will result in indexing errors\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>title</th>\n      <th>heading</th>\n      <th>content</th>\n      <th>tokens</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2020 Summer Olympics</td>\n      <td>Summary</td>\n      <td>The 2020 Summer Olympics (Japanese: 2020\u5e74\u590f\u5b63\u30aa\u30ea\u30f3...</td>\n      <td>713</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2020 Summer Olympics</td>\n      <td>Host city selection</td>\n      <td>The International Olympic Committee (IOC) vote...</td>\n      <td>126</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2020 Summer Olympics</td>\n      <td>Impact of the COVID-19 pandemic</td>\n      <td>In January 2020, concerns were raised about th...</td>\n      <td>369</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2020 Summer Olympics</td>\n      <td>Qualifying event cancellation and postponement</td>\n      <td>Concerns about the pandemic began to affect qu...</td>\n      <td>298</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2020 Summer Olympics</td>\n      <td>Effect on doping tests</td>\n      <td>Mandatory doping tests were being severely res...</td>\n      <td>163</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 321c2a15-b8ad-4113-8300-e7d921a97d94
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"321c2a15-b8ad-4113-8300-e7d921a97d94\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7ea9ff6c1202762e7155acb3717eb1a0e74993130131a336b46f05cfdab5f1d4\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a6bf6803-4365-42cd-aa56-9ea0c7e0beea\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb\", \"title\": \"1. Collect Wikipedia data about Olympic Games 2020\", \"tokens\": 2914, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"45e8d34618c0b72b77bf6765466aae3b4a5a0f33b79d941f9f907544b65107d1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10517, \"end_char_idx\": 12065, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "ref_doc_id": "30bf688b-1c8a-40f4-9ce8-8413a0a6a2d7", "chroma:document": "### Save the section dataset\nWe will save the section dataset, for the [next notebook](olympics-2-create-qa.ipynb)\n\n\n```python\ndf.to_csv('olympics-data/olympics_sections.csv', index=False)\n```\n\n## 1.3 (Optional) Exploring the data \n\n\n```python\ndf.title.value_counts().head()\n```\n\n\n\n\n    Concerns and controversies at the 2020 Summer Olympics    51\n    United States at the 2020 Summer Olympics                 46\n    Great Britain at the 2020 Summer Olympics                 42\n    Canada at the 2020 Summer Olympics                        39\n    Olympic Games                                             39\n    Name: title, dtype: int64\n\n\n\nThere appear to be winter and summer Olympics 2020. We chose to leave a little ambiguity and noise in the dataset, even though we were interested only in the Summer Olympics 2020.\n\n\n```python\ndf.title.str.contains('Summer').value_counts()\n```\n\n\n\n\n    True     3567\n    False     305\n    Name: title, dtype: int64\n\n\n\n\n```python\ndf.title.str.contains('Winter').value_counts()\n```\n\n\n\n\n    False    3774\n    True       98\n    Name: title, dtype: int64\n\n\n\n\n```python\nimport pandas as pd\nfrom matplotlib import pyplot as plt\n\ndf = pd.read_csv('olympics-data/olympics_sections.csv')\ndf[['tokens']].hist()\n# add axis descriptions and title\nplt.xlabel('Number of tokens')\nplt.ylabel('Number of Wikipedia sections')\nplt.title('Distribution of number of tokens in Wikipedia sections')\nplt.show()\n```\n\n\n    \n![png](output_15_0.png)\n    \n\n\nWe can see that the majority of section are fairly short (less than 500 tokens)."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e1dae696-0fc3-4f42-85e2-eea0233f5d7b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e1dae696-0fc3-4f42-85e2-eea0233f5d7b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6e99e081-23cb-4ee8-8fa0-a16c8fafeb8a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f898e720fef8a88b50fc010a9994ac9429bfb03f2224422bc1dda50fe6814fe5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1397, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "<span style=\"color:orange; font-weight:bold\">Note: To answer questions based on text documents, we recommend the procedure in <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering using Embeddings</a>. Some of the code below may rely on <a href=\"https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints\">deprecated API endpoints</a>.</span>\n\n# 2. Creating a synthetic Q&A dataset\nWe use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), a model specialized in following instructions, to create questions based on the given context. Then we also use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta) to answer those questions, given the same context. \n\nThis is expensive, and will also take a long time, as we call the davinci engine for each section. You can simply download the final dataset instead.\n\nWe're using the dataset created using the [previous notebook](olympics-1-collect-data.ipynb)\n\n## 2.1 Read in the data, and create a context\nCreate a context by concatenating the title, the heading and the content of that section\n\n\n```python\nimport pandas as pd\ndf = pd.read_csv('olympics-data/olympics_sections.csv')\ndf['context'] = df.title + \"\\n\" + df.heading + \"\\n\\n\" + df.content\ndf.head()\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6e99e081-23cb-4ee8-8fa0-a16c8fafeb8a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6e99e081-23cb-4ee8-8fa0-a16c8fafeb8a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e1dae696-0fc3-4f42-85e2-eea0233f5d7b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"6a735d22633015e25d9e78263ede3d5beb57dc8604e491531b877509f2125c6b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e43f6807-c153-4603-8454-d4d815c3b3d6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bd95ab2fbdc31bed7ed03497313e3fea2bd8f3253b0bb22457cffb08619c6ff3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3215, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "<span style=\"color:orange; font-weight:bold\">Note: To answer questions based on text documents, we recommend the procedure in <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering using Embeddings</a>. Some of the code below may rely on <a href=\"https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints\">deprecated API endpoints</a>.</span>\n\n# 2. Creating a synthetic Q&A dataset\nWe use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), a model specialized in following instructions, to create questions based on the given context. Then we also use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta) to answer those questions, given the same context. \n\nThis is expensive, and will also take a long time, as we call the davinci engine for each section. You can simply download the final dataset instead.\n\nWe're using the dataset created using the [previous notebook](olympics-1-collect-data.ipynb)\n\n## 2.1 Read in the data, and create a context\nCreate a context by concatenating the title, the heading and the content of that section\n\n\n```python\nimport pandas as pd\ndf = pd.read_csv('olympics-data/olympics_sections.csv')\ndf['context'] = df.title + \"\\n\" + df.heading + \"\\n\\n\" + df.content\ndf.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>title</th>\n      <th>heading</th>\n      <th>content</th>\n      <th>tokens</th>\n      <th>context</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>2020 Summer Olympics</td>\n      <td>Summary</td>\n      <td>The 2020 Summer Olympics (Japanese: 2020\u5e74\u590f\u5b63\u30aa\u30ea\u30f3...</td>\n      <td>713</td>\n      <td>2020 Summer Olympics\\nSummary\\n\\nThe 2020 Summ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2020 Summer Olympics</td>\n      <td>Host city selection</td>\n      <td>The International Olympic Committee (IOC) vote...</td>\n      <td>126</td>\n      <td>2020 Summer Olympics\\nHost city selection\\n\\nT...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2020 Summer Olympics</td>\n      <td>Impact of the COVID-19 pandemic</td>\n      <td>In January 2020, concerns were raised about th...</td>\n      <td>369</td>\n      <td>2020 Summer Olympics\\nImpact of the COVID-19 p...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>2020 Summer Olympics</td>\n      <td>Qualifying event cancellation and postponement</td>\n      <td>Concerns about the pandemic began to affect qu...</td>\n      <td>298</td>\n      <td>2020 Summer Olympics\\nQualifying event cancell...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>2020 Summer Olympics</td>\n      <td>Effect on doping tests</td>\n      <td>Mandatory doping tests were being severely res...</td>\n      <td>163</td>\n      <td>2020 Summer Olympics\\nEffect on doping tests\\n...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e43f6807-c153-4603-8454-d4d815c3b3d6
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e43f6807-c153-4603-8454-d4d815c3b3d6\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6e99e081-23cb-4ee8-8fa0-a16c8fafeb8a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"d60d944bc36e857d7f2b93a57e284652d26f6d0958f33c2cb69352f82097de9d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1e9a90c1-bf3a-455c-bb90-58193dd90ce7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"37ff249c3e5c99e7e601e25d040626d3ec1872124757ebe04272156e33ff6db4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3219, \"end_char_idx\": 5081, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "## 2.2 Create questions based on the context\nUse davinci-instruct to generate a number of plausible questions relating to the Wikipedia section contents.\n\nNote: We have used temperature=0, but it may be beneficial to experiment with a higher temperature to get a higher diversity of questions.\n\n<span style=\"color:orange; font-weight:bold\">WARNING: This step will last a long time, and consume a lot of tokens, as it calls davinci-instruct for every section to generate a number of questions.</span>\n\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n\ndef get_questions(context):\n    try:\n        response = client.chat.completions.create(model=\"davinci-instruct-beta-v3\",\n        prompt=f\"Write questions based on the text below\\n\\nText: {context}\\n\\nQuestions:\\n1.\",\n        temperature=0,\n        max_tokens=257,\n        top_p=1,\n        frequency_penalty=0,\n        presence_penalty=0,\n        stop=[\"\\n\\n\"])\n        return response.choices[0].text\n    except:\n        return \"\"\n\n\ndf['questions']= df.context.apply(get_questions)\ndf['questions'] = \"1.\" + df.questions\nprint(df[['questions']].values[0][0])\n```\n\n    1. What is the 2020 Summer Olympics?\n    2. When did the 2020 Summer Olympics take place?\n    3. Who won the most medals at the 2020 Summer Olympics?\n    4. Who won the most gold medals at the 2020 Summer Olympics?\n    5. Who won the most medals at the 2020 Summer Olympics?\n\n\nThe prompt is designed to generate a number of questions. Example questions above were generated based on the summary section of the 2020 Summer Olympics page.\n\nWe can observe that the questions 3 and 5 above repeat. Sometimes the generated questions could be ambiguous without the context. We will show that even despite these limitations we can create a successful model."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 1e9a90c1-bf3a-455c-bb90-58193dd90ce7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"1e9a90c1-bf3a-455c-bb90-58193dd90ce7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e43f6807-c153-4603-8454-d4d815c3b3d6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"cf436fcf584059b642229f24e698316541da77eb6f6a154772b6e84162cf8e08\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cdc84c4e-2f43-4774-a9ff-01b6a34f2d7e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e0726c13f839c826ae16182175605a6a17f0059ca5f8eeac18e89949dfc481ce\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3721, \"end_char_idx\": 8249, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n\ndef get_questions(context):\n    try:\n        response = client.chat.completions.create(model=\"davinci-instruct-beta-v3\",\n        prompt=f\"Write questions based on the text below\\n\\nText: {context}\\n\\nQuestions:\\n1.\",\n        temperature=0,\n        max_tokens=257,\n        top_p=1,\n        frequency_penalty=0,\n        presence_penalty=0,\n        stop=[\"\\n\\n\"])\n        return response.choices[0].text\n    except:\n        return \"\"\n\n\ndf['questions']= df.context.apply(get_questions)\ndf['questions'] = \"1.\" + df.questions\nprint(df[['questions']].values[0][0])\n```\n\n    1. What is the 2020 Summer Olympics?\n    2. When did the 2020 Summer Olympics take place?\n    3. Who won the most medals at the 2020 Summer Olympics?\n    4. Who won the most gold medals at the 2020 Summer Olympics?\n    5. Who won the most medals at the 2020 Summer Olympics?\n\n\nThe prompt is designed to generate a number of questions. Example questions above were generated based on the summary section of the 2020 Summer Olympics page.\n\nWe can observe that the questions 3 and 5 above repeat. Sometimes the generated questions could be ambiguous without the context. We will show that even despite these limitations we can create a successful model.\n\n\n```python\nprint(df.content.values[0])\n```\n\n    The 2020 Summer Olympics (Japanese: 2020\u5e74\u590f\u5b63\u30aa\u30ea\u30f3\u30d4\u30c3\u30af, Hepburn: Nisen Nij\u016b-nen Kaki Orinpikku), officially the Games of the XXXII Olympiad (\u7b2c\u4e09\u5341\u4e8c\u56de\u30aa\u30ea\u30f3\u30d4\u30c3\u30af\u7af6\u6280\u5927\u4f1a, Dai Sanj\u016bni-kai Orinpikku Ky\u014dgi Taikai) and branded as Tokyo 2020 (\u6771\u4eac2020, T\u014dky\u014d Nii Zero Nii Zero), was an international multi-sport event held from 23 July to 8 August 2021 in Tokyo, Japan, with some preliminary events that began on 21 July.\n    Tokyo was selected as the host city during the 125th IOC Session in Buenos Aires, Argentina, on 7 September 2013. Originally scheduled to take place from 24 July to 9 August 2020, the event was postponed to 2021 in March 2020 as a result of the COVID-19 pandemic, the first such instance in the history of the Olympic Games (previous games had been cancelled but not rescheduled). However, the event retained the Tokyo 2020 name for marketing and branding purposes. It was largely held behind closed doors with no public spectators permitted due to the declaration of a state of emergency in the Greater Tokyo Area in response to the pandemic. The Summer Paralympics were held between 24 August and 5 September 2021, 16 days after the completion of the Olympics.The 2020 Games were the fourth Olympic Games to be held in Japan, following the Tokyo 1964 (Summer), Sapporo 1972 (Winter) and Nagano 1998 (Winter) games. Tokyo is the first city in Asia to hold the Summer Games twice. The 2020 Games were the second of three consecutive Olympics to be held in East Asia, following the 2018 Winter Olympics in Pyeongchang, South Korea and preceding the 2022 Winter Olympics in Beijing, China.\n    New events were introduced in existing sports for 2020, including 3x3 basketball, freestyle BMX and mixed gender team events in a number of existing sports, as well as the return of madison cycling for men and an introduction of the same event for women. New IOC policies also allowed the host organizing committee to add new sports to the Olympic program for just one Games. The disciplines added by the Japanese Olympic Committee were baseball and softball, karate, sport climbing, surfing and skateboarding, the last four of which made their Olympic debuts, and the last three of which will remain on the Olympic program.The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88). Host nation Japan finished third, setting a record for the most gold medals and total medals ever won by their delegation at an Olympic Games with 27 and 58. Great Britain finished fourth, with a total of 22 gold and 65 medals, becoming the first nation at the Summer Olympics to increase or equal their total medals won in the two Games subsequent to hosting them. The Russian delegation competing as the ROC (not to be confused with the Republic of China (Taiwan) which competed as Chinese Taipei, not ROC) finished fifth with 20 gold medals and third in the overall medal count, with 71 medals. Bermuda, the Philippines and Qatar won their first-ever Olympic gold medals. Burkina Faso, San Marino and Turkmenistan won their first-ever Olympic medals."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: cdc84c4e-2f43-4774-a9ff-01b6a34f2d7e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"cdc84c4e-2f43-4774-a9ff-01b6a34f2d7e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1e9a90c1-bf3a-455c-bb90-58193dd90ce7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"d548f2cc12df3e3b7a6eba71d9b39b522601e72923332c318020f7573b637757\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ec54d880-8559-49cb-bc0d-5771c86e3d23\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e2d82bcea0391beeace8fa174d127e5a91d9890e89be86df52aab2755179c96d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8252, \"end_char_idx\": 11280, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "## 2.3 Create answers based on the context\nUse davinci-instruct to answer the questions given the relevant Wikipedia section contents\n\nNote: We have used temperature=0, but it may be beneficial to experiment with a higher temperature to get a higher diversity of questions.\n\n<span style=\"color:orange\">**WARNING: This step will last a long time, and consume a lot of tokens, as it calls davinci-instruct for every section to answer all the questions.**</span>\n\n\n```python\ndef get_answers(row):\n    try:\n        response = client.chat.completions.create(\n            engine=\"davinci-instruct-beta-v3\",\n            prompt=f\"Write answer based on the text below\\n\\nText: {row.context}\\n\\nQuestions:\\n{row.questions}\\n\\nAnswers:\\n1.\",\n            temperature=0,\n            max_tokens=257,\n            top_p=1,\n            frequency_penalty=0,\n            presence_penalty=0\n        )\n        return response.choices[0].text\n    except Exception as e:\n        print (e)\n        return \"\"\n\n\ndf['answers']= df.apply(get_answers, axis=1)\ndf['answers'] = \"1.\" + df.answers\ndf = df.dropna().reset_index().drop('index',axis=1)\nprint(df[['answers']].values[0][0])\n```\n\n    1. The 2020 Summer Olympics is an international multi-sport event held from 23 July to 8 August 2021 in Tokyo, Japan.\n    2. The 2020 Summer Olympics took place from 23 July to 8 August 2021.\n    3. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).\n    4. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).\n    5. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).\n\n\nThese are the answers to the questions above based on the context around the host city selection. \n\nWe can see that answers 3-5 contain the correct answer, but instead of answering the question directly, the answer is a verbatim extraction. Despite these occasional lower quality answers, we will show that the model can learn the task reasonably well, given a high number of examples.\n\n## 2.4 Save the Olympics Q&A dataset based on Wikipedia sections\nWe save the file for use in the [next notebook](olympics-3-train-qa.ipynb)\n\n\n```python\ndf.to_csv('olympics-data/olympics_qa.csv', index=False)\n```\n\n## 2.5 Search file (DEPRECATED)\nWe create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.\n\n<span style=\"color:orange; font-weight:bold\">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering Guide</a> for a search implementation using the embeddings</span>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ec54d880-8559-49cb-bc0d-5771c86e3d23
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ec54d880-8559-49cb-bc0d-5771c86e3d23\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cdc84c4e-2f43-4774-a9ff-01b6a34f2d7e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9cf622602c2607d3253a740f647ea1035019f56db5c1ef51d6068c11f8461d8c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0ffb8141-6bab-4b90-969e-bc1fcbece6b5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b26160b2fc3dd4b93c22d0e38b68ab6b5ca5deab448fedc70b44d3852594faa9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10073, \"end_char_idx\": 12028, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "These are the answers to the questions above based on the context around the host city selection. \n\nWe can see that answers 3-5 contain the correct answer, but instead of answering the question directly, the answer is a verbatim extraction. Despite these occasional lower quality answers, we will show that the model can learn the task reasonably well, given a high number of examples.\n\n## 2.4 Save the Olympics Q&A dataset based on Wikipedia sections\nWe save the file for use in the [next notebook](olympics-3-train-qa.ipynb)\n\n\n```python\ndf.to_csv('olympics-data/olympics_qa.csv', index=False)\n```\n\n## 2.5 Search file (DEPRECATED)\nWe create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.\n\n<span style=\"color:orange; font-weight:bold\">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering Guide</a> for a search implementation using the embeddings</span>\n\n\n\n```python\ndf = df[df.tokens<2000]\ndf[['context', 'tokens']].rename(columns={'context':'text','tokens':'metadata'}).to_json('olympics-data/olympics_search.jsonl', orient='records', lines=True)\n\nsearch_file = client.files.create(\n  file=open(\"olympics-data/olympics_search.jsonl\"),\n  purpose='search'\n)\nolympics_search_fileid = search_file['id']\n```\n\n## 2.6 Answer questions based on the context provided\n\nWe will use a simple implementation of the answers endpoint. This works by simply using the [/search endpoint](https://beta.openai.com/docs/api-reference/searches), which searches over an indexed file to obtain the relevant sections which can be included in the context, following by a question and answering prompt given a specified model."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0ffb8141-6bab-4b90-969e-bc1fcbece6b5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0ffb8141-6bab-4b90-969e-bc1fcbece6b5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ec54d880-8559-49cb-bc0d-5771c86e3d23\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"7d97047947889f466f9a82056ec82541870568e1b73f65bb2529b4709256e370\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"45022073-0536-4298-a178-7e22e52a4962\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2f0267626d95ae41aff474e162dacff7692b380bdd626554cae4e525be23dca0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10602, \"end_char_idx\": 14076, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "```python\ndf.to_csv('olympics-data/olympics_qa.csv', index=False)\n```\n\n## 2.5 Search file (DEPRECATED)\nWe create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.\n\n<span style=\"color:orange; font-weight:bold\">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href=\"https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb\">Question Answering Guide</a> for a search implementation using the embeddings</span>\n\n\n\n```python\ndf = df[df.tokens<2000]\ndf[['context', 'tokens']].rename(columns={'context':'text','tokens':'metadata'}).to_json('olympics-data/olympics_search.jsonl', orient='records', lines=True)\n\nsearch_file = client.files.create(\n  file=open(\"olympics-data/olympics_search.jsonl\"),\n  purpose='search'\n)\nolympics_search_fileid = search_file['id']\n```\n\n## 2.6 Answer questions based on the context provided\n\nWe will use a simple implementation of the answers endpoint. This works by simply using the [/search endpoint](https://beta.openai.com/docs/api-reference/searches), which searches over an indexed file to obtain the relevant sections which can be included in the context, following by a question and answering prompt given a specified model.\n\n\n```python\nfrom answers_with_ft import create_context, answer_question\nprint(create_context(\"Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?\", olympics_search_fileid, max_len=400))\n```\n\n    Athletics at the 2020 Summer Olympics \u2013 Women's 4 \u00d7 100 metres relay\n    Summary\n    \n    The women's 4 \u00d7 100 metres relay event at the 2020 Summer Olympics took place on 5 and 6 August 2021 at the Japan National Stadium. There were 16 competing relay teams, with each team having 5 members from which 4 were selected in each round.\n    \n    ###\n    \n    Athletics at the 2020 Summer Olympics \u2013 Men's 4 \u00d7 100 metres relay\n    Qualification\n    \n    National Olympic Committees (NOCs) could qualify one relay team in one of three following ways:\n    The top 8 NOCs at the 2019 World Athletics Championships qualified a relay team.\n    The top 8 NOCs at the 2021 World Athletics Relays qualified a relay team.\n    Where an NOC placed in the top 8 at both the 2019 World Championships and the 2021 World Relays, the quota place was allocated to the world top list as of 29 June 2021. In this case, 4 teams did so, so there are 4 places available through the world rankings.A total of five athletes may be entered for a relay team. Should a NOC have also entered individual athletes in the corresponding individual event (100 m), the entered individual athletes must be included in the total of five (5) athletes entered for the relay event. In addition of five, NOCs can nominate a maximum of one alternate athlete for each team.\n    The qualifying period was originally from 1 May 2019 to 29 June 2020. Due to the COVID-19 pandemic, the period was suspended from 6 April 2020 to 30 November 2020, with the end date extended to 29 June 2021. The qualifying time standards could be obtained in various meets during the given period that have the approval of the IAAF. Both indoor and outdoor meets are eligible. The most recent Area Championships may be counted in the ranking, even if not during the qualifying period."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 45022073-0536-4298-a178-7e22e52a4962
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"45022073-0536-4298-a178-7e22e52a4962\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0ffb8141-6bab-4b90-969e-bc1fcbece6b5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9fcd86934fc9d29d04399771233349e9a941a5b1f6702df6b8936e09b15b6d50\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3d6b2832-b1d6-4a3a-bdf5-cdff5f08ae8c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e162a1ea5a2e13a63d47a7a226af8464a6db998ee2dfe66f4371b7ccdf6096d5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14080, \"end_char_idx\": 15448, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "```python\nanswer_question(olympics_search_fileid, \"davinci-instruct-beta-v3\", \n            \"Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?\")\n```\n\n\n\n\n    ' Japan National Stadium'\n\n\n\nAfter we fine-tune the model for Q&A we'll be able to use it instead of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), to obtain better answers when the question can't be answered based on the context. We see a downside of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), which always attempts to answer the question, regardless of the relevant context being present or not. (Note the second question is asking about a future event, set in 2024.)\n\n\n```python\nanswer_question(olympics_search_fileid, \"davinci-instruct-beta-v3\", \n            \"Where did women's 4 x 100 metres relay event take place during the 2048 Summer Olympics?\", max_len=1000)\n```\n\n\n\n\n    ' Japan National Stadium'\n\n\n\nWe can see that davinci has a tendency to answer the question, even if the question can't be answered given the context provided. Note the question asked regarding 2048 Summer Olympics, which didn't happen yet, and the retrieved content has only returned results for 2020.\n\n## 2.7 (Optional) Investigation into how likely the search endpoint is to return the relevant context"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 3d6b2832-b1d6-4a3a-bdf5-cdff5f08ae8c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"3d6b2832-b1d6-4a3a-bdf5-cdff5f08ae8c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"45022073-0536-4298-a178-7e22e52a4962\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"89c66e22e494442c292c09d1c4d22d57be09fb649104880b24a7cbb2f5d62f8e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"85190d63-ff96-4fba-976b-c1ee493f1348\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c69dba5798852c1c363dc15b6b742c21b38c995a565652d205b7fd4e889cb418\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14080, \"end_char_idx\": 17343, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "```python\nanswer_question(olympics_search_fileid, \"davinci-instruct-beta-v3\", \n            \"Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?\")\n```\n\n\n\n\n    ' Japan National Stadium'\n\n\n\nAfter we fine-tune the model for Q&A we'll be able to use it instead of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), to obtain better answers when the question can't be answered based on the context. We see a downside of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), which always attempts to answer the question, regardless of the relevant context being present or not. (Note the second question is asking about a future event, set in 2024.)\n\n\n```python\nanswer_question(olympics_search_fileid, \"davinci-instruct-beta-v3\", \n            \"Where did women's 4 x 100 metres relay event take place during the 2048 Summer Olympics?\", max_len=1000)\n```\n\n\n\n\n    ' Japan National Stadium'\n\n\n\nWe can see that davinci has a tendency to answer the question, even if the question can't be answered given the context provided. Note the question asked regarding 2048 Summer Olympics, which didn't happen yet, and the retrieved content has only returned results for 2020.\n\n## 2.7 (Optional) Investigation into how likely the search endpoint is to return the relevant context\n\n\n```python\ndef check_context(title, heading, question, max_len=1800, search_model='ada', max_rerank=10):\n    \"\"\"\n    Evaluate the performance of the search model in retrieving the correct context\n\n    Parameters\n    ----------\n    title: str\n        The title of the Wikipedia page\n    heading: str\n        The heading of the Wikipedia section\n    qusetion: str\n        The question\n    max_len: int\n        The maximum length of the context\n    search_model: str\n        The search model to use - `ada` is most cost effective\n    max_rerank: int\n        The maximum number of reranking documents to use the search model on\n\n    Returns\n    -------\n    rank: int\n        The rank of the correct context\n    token_length: int\n        The number of tokens needed to obtain the correct context\n    \"\"\"\n    \n    try:\n        # TODO: openai.Engine(search_model) is deprecated\n        results = openai.Engine(search_model).search(\n            search_model=search_model, \n            query=question, \n            max_rerank=max_rerank,\n            file=olympics_search_fileid,\n            return_metadata=True\n        )\n        index=-1\n        returns = []\n        cur_len = 0\n        for result in results['data']:\n            cur_len += int(result['metadata']) + 4 # we add 4 tokens for the separator `\\n\\n###\\n\\n`\n            if cur_len > max_len:\n                break\n            returns.append(result['text'])\n            res = result['text'].split('\\n')\n            if res[0] == title and res[1] == heading:\n                index = len(returns) - 1\n                break\n        return index, cur_len\n    except Exception as e:\n        #print (e)\n        return []\nprint(check_context(\"Athletics at the 2020 Summer Olympics \u2013 Women's 4 \u00d7 100 metres relay\", \"Summary\", \"Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?\", max_len=10000))\n```\n\n    (0, 58)"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 85190d63-ff96-4fba-976b-c1ee493f1348
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"85190d63-ff96-4fba-976b-c1ee493f1348\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3d6b2832-b1d6-4a3a-bdf5-cdff5f08ae8c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"8bd676687634852a393618a379893324df0f97d472cc602aa9022ca3cbe3f643\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2c0cce0e-fb26-4f3e-98a4-550d80dd05a3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"068a6b7ab450d88ef40c83ff16b16f5c60058205e74bc92522d5612de1a2dfa8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17346, \"end_char_idx\": 20158, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "We utilize the generated questions based on context to estimate how often we can retrieve the original context. These questions are noisy, so this is not a perfect estimate.\n\nOur questions and answers are prefixed with numbered bullet points, however due to the way they were generated, they are missing the first number, hence we add \"1.\" to the list of questions (and answers).\n\nWe calculate the rank of the section retrieved using ada search, and the number of tokens in the context needed to retrieve the relevant section in full.\n\n\n```python\nada_results = df.apply(lambda x: [\n                    check_context( x.title, \n                                   x.heading, \n                                   q[3:],     # remove the number prefix\n                                   max_len=1000000, # set a large number to get the full context \n                                   search_model='ada', \n                                   max_rerank=200,\n                                 ) \n                    for q in (x.questions).split('\\n') # split the questions\n                    if len(q) >10 # remove the empty questions\n                ], axis=1)\nada_results.head()\n```\n\n\n\n\n    0    [(132, 27104), (-1, 22939), (8, 2151), (2, 121...\n    1    [(4, 1737), (0, 130), (8, 744), (96, 17208), (...\n    2          [(0, 373), (0, 373), (-1, 40610), (1, 570)]\n    3            [(0, 302), (0, 302), (5, 968), (8, 1425)]\n    4                      [(0, 167), (0, 167), (2, 1442)]\n    Name: ada, dtype: object\n\n\n\n\n```python\nout = pd.concat([ada_results], axis=1)\nout.columns = ['ada']\nout.to_csv('olympics-data/search_engine_results.csv')\n```\n\n\n```python\ndef expand_lists(out):\n    \"\"\"\n    Expand a pandas series containing lists into a series, where each list element becomes a value on its own\n\n    Input is a row per paragraph, which has multiple questions\n    Output is a row per question\n    \"\"\"\n    cols = [pd.DataFrame(out[name].tolist()).stack().reset_index(level=1, drop=True).rename(name) for name in out.columns] \n    return pd.concat(cols, axis=1)\n\nout_expanded = expand_lists(out)\nout_expanded['rank'] = out_expanded.ada.apply(lambda x: x[0] if x != [] else -2)\nout_expanded['tokens'] = out_expanded.ada.apply(lambda x: x[1] if x != [] else -2)\n\n```\n\n\n```python\nwithin_2k = (out_expanded.tokens < 2000).mean()\nprint(f\"{within_2k*100:.1f}% of relevant paragraphs are retrieved within the first 2k tokens\")\n```\n\n    74.3% of relevant paragraphs are retrieved within the first 2k tokens\n\n\nThe relevant context can be obtained 74% of the time on this dataset\n\n\n```python\noutside_200 = (out_expanded['rank'] == -1).mean()\nprint(f\"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results\")\n```\n\n    7.4% of relevant paragraphs are not retrieved within the first 200 results"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2c0cce0e-fb26-4f3e-98a4-550d80dd05a3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2c0cce0e-fb26-4f3e-98a4-550d80dd05a3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"85190d63-ff96-4fba-976b-c1ee493f1348\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b2999c2b87a8b26ccbd9eb0d69a8327acc499eb67f160ce82bd6ba16f315601b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"850aedcc-ac8e-4815-828c-259c2d9e05fc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"062d44d0693558cf08d89eb8e656f1b0c99a05740d3b694619f9f64e1b71fe6e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18856, \"end_char_idx\": 21286, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "```python\nout = pd.concat([ada_results], axis=1)\nout.columns = ['ada']\nout.to_csv('olympics-data/search_engine_results.csv')\n```\n\n\n```python\ndef expand_lists(out):\n    \"\"\"\n    Expand a pandas series containing lists into a series, where each list element becomes a value on its own\n\n    Input is a row per paragraph, which has multiple questions\n    Output is a row per question\n    \"\"\"\n    cols = [pd.DataFrame(out[name].tolist()).stack().reset_index(level=1, drop=True).rename(name) for name in out.columns] \n    return pd.concat(cols, axis=1)\n\nout_expanded = expand_lists(out)\nout_expanded['rank'] = out_expanded.ada.apply(lambda x: x[0] if x != [] else -2)\nout_expanded['tokens'] = out_expanded.ada.apply(lambda x: x[1] if x != [] else -2)\n\n```\n\n\n```python\nwithin_2k = (out_expanded.tokens < 2000).mean()\nprint(f\"{within_2k*100:.1f}% of relevant paragraphs are retrieved within the first 2k tokens\")\n```\n\n    74.3% of relevant paragraphs are retrieved within the first 2k tokens\n\n\nThe relevant context can be obtained 74% of the time on this dataset\n\n\n```python\noutside_200 = (out_expanded['rank'] == -1).mean()\nprint(f\"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results\")\n```\n\n    7.4% of relevant paragraphs are not retrieved within the first 200 results\n\n\n7.4% of the time, this is due to the keyword search part of the search algorithm not retrieving the relevant context within the first 200 results.\n18.3% of the time this is due to the semantic search not placing the relevant context within the first 2000 tokens.\n\n\n```python\nimport matplotlib.pyplot as plt\n\n# plot a histogram, and add axis descriptions and title\nout_expanded[(out_expanded['rank'] >=0)&(out_expanded['rank'] <30)]['rank'].hist(bins=29)\nplt.xlabel('rank')\nplt.ylabel('count')\nplt.title('Histogram of ranks of retrieved paragraphs')\nplt.show()\n```\n\n\n    \n![png](output_31_0.png)\n    \n\n\n\n```python\nout_expanded[(out_expanded.tokens>=0)&(out_expanded.tokens < 2000)]['tokens'].hist(bins=29)\nplt.xlabel('tokens')\nplt.ylabel('count')\nplt.title('Histogram of the number of minimum tokens needed')\nplt.show()\n```\n\n\n    \n![png](output_32_0.png)\n    \n\n\nWe can observe that the context is most likely to be returned as one of the first results, and most likely to be returned within the first 200-500 tokens.\n\n\n```python\n# normalized value_counts\nout_expanded['rank'].value_counts(normalize=True).sort_index()[:13]\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 850aedcc-ac8e-4815-828c-259c2d9e05fc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"850aedcc-ac8e-4815-828c-259c2d9e05fc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"da500b3c-9093-4acb-af8a-9bdf5a59d788\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2c0cce0e-fb26-4f3e-98a4-550d80dd05a3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb\", \"title\": \"2. Creating a synthetic Q&A dataset\", \"tokens\": 5416, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"a00993fda6ae1f8d6c8c7836490516b49574f6ec100efc9b34bfb90cb2e562b2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19841, \"end_char_idx\": 21708, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "ref_doc_id": "da500b3c-9093-4acb-af8a-9bdf5a59d788", "chroma:document": "The relevant context can be obtained 74% of the time on this dataset\n\n\n```python\noutside_200 = (out_expanded['rank'] == -1).mean()\nprint(f\"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results\")\n```\n\n    7.4% of relevant paragraphs are not retrieved within the first 200 results\n\n\n7.4% of the time, this is due to the keyword search part of the search algorithm not retrieving the relevant context within the first 200 results.\n18.3% of the time this is due to the semantic search not placing the relevant context within the first 2000 tokens.\n\n\n```python\nimport matplotlib.pyplot as plt\n\n# plot a histogram, and add axis descriptions and title\nout_expanded[(out_expanded['rank'] >=0)&(out_expanded['rank'] <30)]['rank'].hist(bins=29)\nplt.xlabel('rank')\nplt.ylabel('count')\nplt.title('Histogram of ranks of retrieved paragraphs')\nplt.show()\n```\n\n\n    \n![png](output_31_0.png)\n    \n\n\n\n```python\nout_expanded[(out_expanded.tokens>=0)&(out_expanded.tokens < 2000)]['tokens'].hist(bins=29)\nplt.xlabel('tokens')\nplt.ylabel('count')\nplt.title('Histogram of the number of minimum tokens needed')\nplt.show()\n```\n\n\n    \n![png](output_32_0.png)\n    \n\n\nWe can observe that the context is most likely to be returned as one of the first results, and most likely to be returned within the first 200-500 tokens.\n\n\n```python\n# normalized value_counts\nout_expanded['rank'].value_counts(normalize=True).sort_index()[:13]\n```\n\n\n\n\n    -2     0.000063\n    -1     0.074428\n     0     0.453420\n     1     0.089515\n     2     0.047146\n     3     0.032437\n     4     0.024139\n     5     0.019676\n     6     0.015967\n     7     0.013452\n     8     0.011189\n     9     0.009869\n     10    0.009178\n    Name: rank, dtype: float64\n\n\n\nprobabilities of the relevant context being returned at each rank. (-2 means a processing error, -1 means the rank is >200)"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: da298b28-f39e-41b7-97b4-213b9a918b83
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"da298b28-f39e-41b7-97b4-213b9a918b83\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0aa5bf93-3c4c-428b-9e66-66d151e7975a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"bd6309d85754359747104e4d5a7c9b5f0a223073ce49b3d0e67a24c91449ebb3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3009, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "# Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\n\nThe aim of this notebook is to walk through a comprehensive example of how to fine-tune OpenAI models for Retrieval Augmented Generation (RAG). \n\nWe will also be integrating Qdrant and Few-Shot Learning to boost the model's performance and reduce hallucinations. This could serve as a practical guide for ML practitioners, data scientists, and AI Engineers interested in leveraging the power of OpenAI models for specific use-cases. \ud83e\udd29\n\n## Why should you read this blog?\n\nYou want to learn how to \n- [Fine-tune OpenAI models](https://platform.openai.com/docs/guides/fine-tuning/) for specific use-cases\n- Use [Qdrant](https://qdrant.tech/documentation/) to improve the performance of your RAG model\n- Use fine-tuning to improve the correctness of your RAG model and reduce hallucinations\n\nTo begin, we've selected a dataset where we've a guarantee that the retrieval is perfect. We've selected a subset of the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset, which is a collection of questions and answers about Wikipedia articles. We've also included samples where the answer is not present in the context, to demonstrate how RAG handles this case.\n\n## Table of Contents\n1. Setting up the Environment\n\n### Section A: Zero-Shot Learning\n2. Data Preparation: SQuADv2 Dataset\n3. Answering using Base gpt-3.5-turbo-0613 model\n4. Fine-tuning and Answering using Fine-tuned model\n5. **Evaluation**: How well does the model perform?\n\n### Section B: Few-Shot Learning\n\n6. Using Qdrant to Improve RAG Prompt\n7. Fine-Tuning OpenAI Model with Qdrant\n8. Evaluation\n\n9. **Conclusion**\n    - Aggregate Results\n    - Observations\n\n## Terms, Definitions, and References\n\n**Retrieval Augmented Generation (RAG)?**\nThe phrase Retrieval Augmented Generation (RAG) comes from a [recent paper](https://arxiv.org/abs/2005.11401) by Lewis et al. from Facebook AI. The idea is to use a pre-trained language model (LM) to generate text, but to use a separate retrieval system to find relevant documents to condition the LM on. \n\n**What is Qdrant?**\nQdrant is an open-source vector search engine that allows you to search for similar vectors in a large dataset. It is built in Rust and here we'll use the Python client to interact with it. This is the Retrieval part of RAG.\n\n**What is Few-Shot Learning?**\nFew-shot learning is a type of machine learning where the model is \"improved\" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.\n\n**What is Zero-Shot Learning?**\nZero-shot learning is a type of machine learning where the model is \"improved\" via training or fine-tuning without any dataset specific information. \n\n**What is Fine-Tuning?**\nFine-tuning is a type of machine learning where the model is \"improved\" via training or fine-tuning on a small amount of data."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0aa5bf93-3c4c-428b-9e66-66d151e7975a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0aa5bf93-3c4c-428b-9e66-66d151e7975a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"da298b28-f39e-41b7-97b4-213b9a918b83\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"61118c1d33336410355df7f7601489f39754bb03adc12f3fb442d2a6a79312b9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"75819f84-aa1f-414b-a3fb-f51340a02864\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"19882e9be2e5a9df2b233c2312f3da3ffc7779d8b9286d05594df5ddda2a7578\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1442, \"end_char_idx\": 4059, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "Fine-tuning and Answering using Fine-tuned model\n5. **Evaluation**: How well does the model perform?\n\n### Section B: Few-Shot Learning\n\n6. Using Qdrant to Improve RAG Prompt\n7. Fine-Tuning OpenAI Model with Qdrant\n8. Evaluation\n\n9. **Conclusion**\n    - Aggregate Results\n    - Observations\n\n## Terms, Definitions, and References\n\n**Retrieval Augmented Generation (RAG)?**\nThe phrase Retrieval Augmented Generation (RAG) comes from a [recent paper](https://arxiv.org/abs/2005.11401) by Lewis et al. from Facebook AI. The idea is to use a pre-trained language model (LM) to generate text, but to use a separate retrieval system to find relevant documents to condition the LM on. \n\n**What is Qdrant?**\nQdrant is an open-source vector search engine that allows you to search for similar vectors in a large dataset. It is built in Rust and here we'll use the Python client to interact with it. This is the Retrieval part of RAG.\n\n**What is Few-Shot Learning?**\nFew-shot learning is a type of machine learning where the model is \"improved\" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.\n\n**What is Zero-Shot Learning?**\nZero-shot learning is a type of machine learning where the model is \"improved\" via training or fine-tuning without any dataset specific information. \n\n**What is Fine-Tuning?**\nFine-tuning is a type of machine learning where the model is \"improved\" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. The LLM is what makes the Generation part of RAG.\n\n## 1. Setting Up the Environment\n\n### Install and Import Dependencies\n\n\n```python\n!pip install pandas openai tqdm tenacity scikit-learn tiktoken python-dotenv seaborn --upgrade --quiet\n```\n\n\n```python\nimport json\nimport os\nimport time\n\nimport pandas as pd\nfrom openai import OpenAI\nimport tiktoken\nimport seaborn as sns\nfrom tenacity import retry, wait_exponential\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.metrics import confusion_matrix\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\ntqdm.pandas()\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n\n```\n\n### Set your keys\nGet your OpenAI keys [here](https://platform.openai.com/account/api-keys) and Qdrant keys after making a free cluster [here](https://cloud.qdrant.io/login)."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 75819f84-aa1f-414b-a3fb-f51340a02864
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"75819f84-aa1f-414b-a3fb-f51340a02864\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0aa5bf93-3c4c-428b-9e66-66d151e7975a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"42613a99e181198e4c06b400f4efb7b57eea9a7720aea409e94e3878bf080343\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d0bba0bf-613a-4d4a-8e3e-695ca7a1775f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7afdd8b85611d4f227050ddcbfda91240c9004a269ef33b06ba69acbac0892b5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2531, \"end_char_idx\": 4967, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.\n\n**What is Zero-Shot Learning?**\nZero-shot learning is a type of machine learning where the model is \"improved\" via training or fine-tuning without any dataset specific information. \n\n**What is Fine-Tuning?**\nFine-tuning is a type of machine learning where the model is \"improved\" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. The LLM is what makes the Generation part of RAG.\n\n## 1. Setting Up the Environment\n\n### Install and Import Dependencies\n\n\n```python\n!pip install pandas openai tqdm tenacity scikit-learn tiktoken python-dotenv seaborn --upgrade --quiet\n```\n\n\n```python\nimport json\nimport os\nimport time\n\nimport pandas as pd\nfrom openai import OpenAI\nimport tiktoken\nimport seaborn as sns\nfrom tenacity import retry, wait_exponential\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.metrics import confusion_matrix\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\ntqdm.pandas()\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n\n```\n\n### Set your keys\nGet your OpenAI keys [here](https://platform.openai.com/account/api-keys) and Qdrant keys after making a free cluster [here](https://cloud.qdrant.io/login).\n\n\n```python\nos.environ[\"QDRANT_URL\"] = \"https://xxx.cloud.qdrant.io:6333\"\nos.environ[\"QDRANT_API_KEY\"] = \"xxx\"\n```\n\n## Section A\n\n## 2. Data Preparation: SQuADv2 Data Subsets\n\nFor the purpose of demonstration, we'll make small slices from the train and validation splits of the [SQuADv2](https://rajpurkar.github.io/SQuAD-explorer/) dataset. This dataset has questions and contexts where the answer is not present in the context, to help us evaluate how LLM handles this case.\n\nWe'll read the data from the JSON files and create a dataframe with the following columns: `question`, `context`, `answer`, `is_impossible`.\n\n### Download the Data\n\n\n```python\n# !mkdir -p local_cache\n# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json -O local_cache/train.json\n# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O local_cache/dev.json\n```\n\n### Read JSON to DataFrame"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d0bba0bf-613a-4d4a-8e3e-695ca7a1775f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d0bba0bf-613a-4d4a-8e3e-695ca7a1775f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"75819f84-aa1f-414b-a3fb-f51340a02864\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"de7283f094a9bf414bd7690cdd73c04ea16d54534dbf794ef19e1fff772d508d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a19f8c2c-444d-4faa-89ee-34f72158fcbe\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a42d924a367cff538724c55c136c21c210e8bfa39cf70b01b6d4e16bc4a83f25\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4062, \"end_char_idx\": 7384, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\nos.environ[\"QDRANT_URL\"] = \"https://xxx.cloud.qdrant.io:6333\"\nos.environ[\"QDRANT_API_KEY\"] = \"xxx\"\n```\n\n## Section A\n\n## 2. Data Preparation: SQuADv2 Data Subsets\n\nFor the purpose of demonstration, we'll make small slices from the train and validation splits of the [SQuADv2](https://rajpurkar.github.io/SQuAD-explorer/) dataset. This dataset has questions and contexts where the answer is not present in the context, to help us evaluate how LLM handles this case.\n\nWe'll read the data from the JSON files and create a dataframe with the following columns: `question`, `context`, `answer`, `is_impossible`.\n\n### Download the Data\n\n\n```python\n# !mkdir -p local_cache\n# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json -O local_cache/train.json\n# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O local_cache/dev.json\n```\n\n### Read JSON to DataFrame\n\n\n```python\ndef json_to_dataframe_with_titles(json_data):\n    qas = []\n    context = []\n    is_impossible = []\n    answers = []\n    titles = []\n\n    for article in json_data['data']:\n        title = article['title']\n        for paragraph in article['paragraphs']:\n            for qa in paragraph['qas']:\n                qas.append(qa['question'].strip())\n                context.append(paragraph['context'])\n                is_impossible.append(qa['is_impossible'])\n                \n                ans_list = []\n                for ans in qa['answers']:\n                    ans_list.append(ans['text'])\n                answers.append(ans_list)\n                titles.append(title)\n\n    df = pd.DataFrame({'title': titles, 'question': qas, 'context': context, 'is_impossible': is_impossible, 'answers': answers})\n    return df\n\ndef get_diverse_sample(df, sample_size=100, random_state=42):\n    \"\"\"\n    Get a diverse sample of the dataframe by sampling from each title\n    \"\"\"\n    sample_df = df.groupby(['title', 'is_impossible']).apply(lambda x: x.sample(min(len(x), max(1, sample_size // 50)), random_state=random_state)).reset_index(drop=True)\n    \n    if len(sample_df) < sample_size:\n        remaining_sample_size = sample_size - len(sample_df)\n        remaining_df = df.drop(sample_df.index).sample(remaining_sample_size, random_state=random_state)\n        sample_df = pd.concat([sample_df, remaining_df]).sample(frac=1, random_state=random_state).reset_index(drop=True)\n\n    return sample_df.sample(min(sample_size, len(sample_df)), random_state=random_state).reset_index(drop=True)\n\ntrain_df = json_to_dataframe_with_titles(json.load(open('local_cache/train.json')))\nval_df = json_to_dataframe_with_titles(json.load(open('local_cache/dev.json')))\n\ndf = get_diverse_sample(val_df, sample_size=100, random_state=42)\n```\n\n## 3. Answering using Base gpt-3.5-turbo-0613 model\n\n### 3.1 Zero Shot Prompt\n\nLet's start by using the base gpt-3.5-turbo-0613 model to answer the questions. This prompt is a simple concatenation of the question and context, with a separator token in between: `\\n\\n`. We've a simple instruction part of the prompt: \n\n> Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.\n\nOther prompts are possible, but this is a good starting point. We'll use this prompt to answer the questions in the validation set."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a19f8c2c-444d-4faa-89ee-34f72158fcbe
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a19f8c2c-444d-4faa-89ee-34f72158fcbe\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d0bba0bf-613a-4d4a-8e3e-695ca7a1775f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"765fa4aa1ab28f05589ee33dac6f19c268514fcb948909da44f2c9fea4e13bea\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8a736ffd-4cba-48f9-bc30-9394a08dace8\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5ec93f2cc416bf59589b037262979f8372d781ef569a34f7ad36309f7d723244\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\n# Function to get prompt messages\ndef get_prompt(row):\n    return [\n        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n        {\n            \"role\": \"user\",\n            \"content\": f\"\"\"Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.\n    Question: {row.question}\\n\\n\n    Context: {row.context}\\n\\n\n    Answer:\\n\"\"\",\n        },\n    ]\n```\n\n### 3.2 Answering using Zero Shot Prompt\n\nNext, you'll need some re-usable functions which make an OpenAI API Call and return the answer. You'll use the `ChatCompletion.create` endpoint of the API, which takes a prompt and returns the completed text.\n\n\n```python\n# Function with tenacity for retries\n@retry(wait=wait_exponential(multiplier=1, min=2, max=6))\ndef api_call(messages, model):\n    return client.chat.completions.create(\n        model=model,\n        messages=messages,\n        stop=[\"\\n\\n\"],\n        max_tokens=100,\n        temperature=0.0,\n    )\n\n\n# Main function to answer question\ndef answer_question(row, prompt_func=get_prompt, model=\"gpt-3.5-turbo\"):\n    messages = prompt_func(row)\n    response = api_call(messages, model)\n    return response.choices[0].message.content\n```\n\n\u23f0 **Time to run: ~3 min**, \ud83d\udedc Needs Internet Connection\n\n\n```python\n# Use progress_apply with tqdm for progress bar\ndf[\"generated_answer\"] = df.progress_apply(answer_question, axis=1)\ndf.to_json(\"local_cache/100_val.json\", orient=\"records\", lines=True)\ndf = pd.read_json(\"local_cache/100_val.json\", orient=\"records\", lines=True)\n```\n\n\n```python\ndf\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>title</th>\n      <th>question</th>\n      <th>context</th>\n      <th>is_impossible</th>\n      <th>answers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Scottish_Parliament</td>\n      <td>What consequence of establishing the Scottish .</td>\n      <td>A procedural consequence of the establishment .</td>\n      <td>False</td>\n      <td>[able to vote on domestic legislation that app.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Imperialism</td>\n      <td>Imperialism is less often associated with whic.</td>\n      <td>The principles of imperialism are often genera.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Economic_inequality</td>\n      <td>What issues can't prevent women from working o.</td>\n      <td>When a person\u2019s capabilities are lowered, they."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 8a736ffd-4cba-48f9-bc30-9394a08dace8
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"8a736ffd-4cba-48f9-bc30-9394a08dace8\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a19f8c2c-444d-4faa-89ee-34f72158fcbe\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"cc61c7e97b40e65e9fd44e397cdb220ba4f1566393097d81084d00257d0fc808\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2ca55eb5-7756-46a9-867e-609b179e2a95\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"efbbedffe65f71d470b5caef81673f53f61d570a03f7d2c364ec3ec1a03c3694\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\ndf\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>title</th>\n      <th>question</th>\n      <th>context</th>\n      <th>is_impossible</th>\n      <th>answers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Scottish_Parliament</td>\n      <td>What consequence of establishing the Scottish .</td>\n      <td>A procedural consequence of the establishment .</td>\n      <td>False</td>\n      <td>[able to vote on domestic legislation that app.</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Imperialism</td>\n      <td>Imperialism is less often associated with whic.</td>\n      <td>The principles of imperialism are often genera.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Economic_inequality</td>\n      <td>What issues can't prevent women from working o.</td>\n      <td>When a person\u2019s capabilities are lowered, they.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Southern_California</td>\n      <td>What county are Los Angeles, Orange, San Diego.</td>\n      <td>Its counties of Los Angeles, Orange, San Diego.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>French_and_Indian_War</td>\n      <td>When was the deportation of Canadians?</td>\n      <td>Britain gained control of French Canada and Ac.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>.</th>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n    </tr>\n    <tr>\n      <th>95</th>\n      <td>Geology</td>\n      <td>In the layered Earth model, what is the inner .</td>\n      <td>Seismologists can use the arrival times of sei.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>Prime_number</td>\n      <td>What type of value would the Basel function ha.</td>\n      <td>The zeta function is closely related to prime .</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>Fresno,"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2ca55eb5-7756-46a9-867e-609b179e2a95
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2ca55eb5-7756-46a9-867e-609b179e2a95\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8a736ffd-4cba-48f9-bc30-9394a08dace8\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"7831e60d017808890151e9f3db8332b39850fe0fc7498744d2ac6832cbfbf2d5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d34d29a2-8a9c-497f-a7fd-e25ed5768a2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e4794ada91c8216cc43a36727992fa6a9e01640642155da2528350b4fd6f811\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "they.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Southern_California</td>\n      <td>What county are Los Angeles, Orange, San Diego.</td>\n      <td>Its counties of Los Angeles, Orange, San Diego.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>French_and_Indian_War</td>\n      <td>When was the deportation of Canadians?</td>\n      <td>Britain gained control of French Canada and Ac.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>.</th>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n      <td>.</td>\n    </tr>\n    <tr>\n      <th>95</th>\n      <td>Geology</td>\n      <td>In the layered Earth model, what is the inner .</td>\n      <td>Seismologists can use the arrival times of sei.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>Prime_number</td>\n      <td>What type of value would the Basel function ha.</td>\n      <td>The zeta function is closely related to prime .</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>Fresno,_California</td>\n      <td>What does the San Joaquin Valley Railroad cros.</td>\n      <td>Passenger rail service is provided by Amtrak S.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>Victoria_(Australia)</td>\n      <td>What party rules in Melbourne's inner regions?</td>\n      <td>The centre-left Australian Labor Party (ALP), .</td>\n      <td>False</td>\n      <td>[The Greens, Australian Greens, Greens]</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>Immune_system</td>\n      <td>The speed of the killing response of the human.</td>\n      <td>In humans, this response is activated by compl.</td>\n      <td>False</td>\n      <td>[signal amplification, signal amplification, s.</td>\n    </tr>\n  </tbody>\n</table>\n<p>100 rows \u00d7 5 columns</p>\n</div>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d34d29a2-8a9c-497f-a7fd-e25ed5768a2c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d34d29a2-8a9c-497f-a7fd-e25ed5768a2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2ca55eb5-7756-46a9-867e-609b179e2a95\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"f27232efe20d552226e4ac24ad7241b125f8ae209afec0ddb844d2fef34c2cea\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2df108ed-d301-4eea-b41f-b33bd95a7b19\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"57ff9005f07cbfaec771da9f4ba06230b120d978ef00ec573a97af03aa38c9f8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "what is the inner .</td>\n      <td>Seismologists can use the arrival times of sei.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>Prime_number</td>\n      <td>What type of value would the Basel function ha.</td>\n      <td>The zeta function is closely related to prime .</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>Fresno,_California</td>\n      <td>What does the San Joaquin Valley Railroad cros.</td>\n      <td>Passenger rail service is provided by Amtrak S.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>Victoria_(Australia)</td>\n      <td>What party rules in Melbourne's inner regions?</td>\n      <td>The centre-left Australian Labor Party (ALP), .</td>\n      <td>False</td>\n      <td>[The Greens, Australian Greens, Greens]</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>Immune_system</td>\n      <td>The speed of the killing response of the human.</td>\n      <td>In humans, this response is activated by compl.</td>\n      <td>False</td>\n      <td>[signal amplification, signal amplification, s.</td>\n    </tr>\n  </tbody>\n</table>\n<p>100 rows \u00d7 5 columns</p>\n</div>\n\n\n\n## 4. Fine-tuning and Answering using Fine-tuned model\n\nFor the complete fine-tuning process, please refer to the [OpenAI Fine-Tuning Docs](https://platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model).\n\n### 4.1 Prepare the Fine-Tuning Data\n\nWe need to prepare the data for fine-tuning. We'll use a few samples from train split of same dataset as before, but we'll add the answer to the context. This will help the model learn to retrieve the answer from the context. \n\nOur instruction prompt is the same as before, and so is the system prompt."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2df108ed-d301-4eea-b41f-b33bd95a7b19
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2df108ed-d301-4eea-b41f-b33bd95a7b19\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d34d29a2-8a9c-497f-a7fd-e25ed5768a2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"0149784cb927f00e9c54f3ef1511595e6dbbb7f74d378c00d532093881ae7a58\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6859f0c4-6246-4ea9-a8bd-9e5bae5c946e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6f74cef954d61cb79d144ad5fdf416e110044fb0132d3092b591f38daf10636c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "_California</td>\n      <td>What does the San Joaquin Valley Railroad cros.</td>\n      <td>Passenger rail service is provided by Amtrak S.</td>\n      <td>True</td>\n      <td>[]</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>Victoria_(Australia)</td>\n      <td>What party rules in Melbourne's inner regions?</td>\n      <td>The centre-left Australian Labor Party (ALP), .</td>\n      <td>False</td>\n      <td>[The Greens, Australian Greens, Greens]</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>Immune_system</td>\n      <td>The speed of the killing response of the human.</td>\n      <td>In humans, this response is activated by compl.</td>\n      <td>False</td>\n      <td>[signal amplification, signal amplification, s.</td>\n    </tr>\n  </tbody>\n</table>\n<p>100 rows \u00d7 5 columns</p>\n</div>\n\n\n\n## 4. Fine-tuning and Answering using Fine-tuned model\n\nFor the complete fine-tuning process, please refer to the [OpenAI Fine-Tuning Docs](https://platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model).\n\n### 4.1 Prepare the Fine-Tuning Data\n\nWe need to prepare the data for fine-tuning. We'll use a few samples from train split of same dataset as before, but we'll add the answer to the context. This will help the model learn to retrieve the answer from the context. \n\nOur instruction prompt is the same as before, and so is the system prompt. \n\n\n```python\ndef dataframe_to_jsonl(df):\n    def create_jsonl_entry(row):\n        answer = row[\"answers\"][0] if row[\"answers\"] else \"I don't know\"\n        messages = [\n            {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n            {\n                \"role\": \"user\",\n                \"content\": f\"\"\"Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.\n            Question: {row.question}\\n\\n\n            Context: {row.context}\\n\\n\n            Answer:\\n\"\"\",\n            },\n            {\"role\": \"assistant\", \"content\": answer},\n        ]\n        return json.dumps({\"messages\": messages})\n\n    jsonl_output = df.apply(create_jsonl_entry, axis=1)\n    return \"\\n\".join(jsonl_output)\n\ntrain_sample = get_diverse_sample(train_df, sample_size=100, random_state=42)\n\nwith open(\"local_cache/100_train.jsonl\", \"w\") as f:\n    f.write(dataframe_to_jsonl(train_sample))\n```\n\n**Tip: \ud83d\udca1 Verify the Fine-Tuning Data**\n\nYou can see this [cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/Chat_finetuning_data_prep.ipynb) for more details on how to prepare the data for fine-tuning.\n\n### 4.2 Fine-Tune OpenAI Model\n\nIf you're new to OpenAI Model Fine-Tuning, please refer to the [How to finetune Chat models](https://github.com/openai/openai-cookbook/blob/448a0595b84ced3bebc9a1568b625e748f9c1d60/examples/How_to_finetune_chat_models.ipynb) notebook. You can also refer to the [OpenAI Fine-Tuning Docs](platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model) for more details."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6859f0c4-6246-4ea9-a8bd-9e5bae5c946e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6859f0c4-6246-4ea9-a8bd-9e5bae5c946e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2df108ed-d301-4eea-b41f-b33bd95a7b19\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"7095abf517940c201bc7be50b51e3d08c9bf4a16ce96e616819aa410215d667b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4049c1f2-2480-4d5a-98da-bfb5d56f5dd3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0578ada283197b1b1d56ba723adb382013efac4a40dee49e006c32e9c5db664e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14308, \"end_char_idx\": 17379, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\nclass OpenAIFineTuner:\n    \"\"\"\n    Class to fine tune OpenAI models\n    \"\"\"\n    def __init__(self, training_file_path, model_name, suffix):\n        self.training_file_path = training_file_path\n        self.model_name = model_name\n        self.suffix = suffix\n        self.file_object = None\n        self.fine_tuning_job = None\n        self.model_id = None\n\n    def create_openai_file(self):\n        self.file_object = client.files.create(\n            file=open(self.training_file_path, \"r\"),\n            purpose=\"fine-tune\",\n        )\n\n    def wait_for_file_processing(self, sleep_time=20):\n        while self.file_object.status != 'processed':\n            time.sleep(sleep_time)\n            self.file_object.refresh()\n            print(\"File Status: \", self.file_object.status)\n\n    def create_fine_tuning_job(self):\n        self.fine_tuning_job = client.fine_tuning.jobs.create(\n            training_file=self.file_object[\"id\"],\n            model=self.model_name,\n            suffix=self.suffix,\n        )\n\n    def wait_for_fine_tuning(self, sleep_time=45):\n        while self.fine_tuning_job.status != 'succeeded':\n            time.sleep(sleep_time)\n            self.fine_tuning_job.refresh()\n            print(\"Job Status: \", self.fine_tuning_job.status)\n\n    def retrieve_fine_tuned_model(self):\n        self.model_id = client.fine_tuning.jobs.retrieve(self.fine_tuning_job[\"id\"]).fine_tuned_model\n        return self.model_id\n\n    def fine_tune_model(self):\n        self.create_openai_file()\n        self.wait_for_file_processing()\n        self.create_fine_tuning_job()\n        self.wait_for_fine_tuning()\n        return self.retrieve_fine_tuned_model()\n\nfine_tuner = OpenAIFineTuner(\n        training_file_path=\"local_cache/100_train.jsonl\",\n        model_name=\"gpt-3.5-turbo\",\n        suffix=\"100trn20230907\"\n    )\n```\n\n\u23f0 **Time to run: ~10-20 minutes**, \ud83d\udedc Needs Internet Connection\n\n\n```python\nmodel_id = fine_tuner.fine_tune_model()\nmodel_id\n```\n\n#### 4.2.1 Try out the Fine-Tuned Model\n\nLet's try out the fine-tuned model on the same validation set as before. You'll use the same prompt as before, but you will use the fine-tuned model instead of the base model. Before you do that, you can make a simple call to get a sense of how the fine-tuned model is doing.\n\n\n```python\ncompletion = client.chat.completions.create(\n    model=model_id,\n    messages=[\n        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n        {\"role\": \"user\", \"content\": \"Hello!\"},\n        {\"role\": \"assistant\", \"content\": \"Hi, how can I help you today?\"},\n        {\n            \"role\": \"user\",\n            \"content\": \"Can you answer the following question based on the given context? If not, say, I don't know:\\n\\nQuestion: What is the capital of France?\\n\\nContext: The capital of Mars is Gaia. Answer:\",\n        },\n    ],\n)\n\nprint(completion.choices[0].message)\n```\n\n### 4.3 Answer Using the Fine-Tuned Model\n\nThis is the same as before, but you'll use the fine-tuned model instead of the base model.\n\n\u23f0 **Time to run: ~5 min**, \ud83d\udedc Needs Internet Connection"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 4049c1f2-2480-4d5a-98da-bfb5d56f5dd3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4049c1f2-2480-4d5a-98da-bfb5d56f5dd3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6859f0c4-6246-4ea9-a8bd-9e5bae5c946e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"a6a2d9f32a06e425ac72ac1b1126ef0d0ff325a2bb069784b19e0acdfda1b15f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ba66d43c-e36b-4c8a-9fee-4a1c8f8381a7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"72fc6e7c8ac5cfa9916d6ea58a4fa655ae496f8e23a6b2f0de42a9ec376ba68a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16211, \"end_char_idx\": 19255, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\nmodel_id = fine_tuner.fine_tune_model()\nmodel_id\n```\n\n#### 4.2.1 Try out the Fine-Tuned Model\n\nLet's try out the fine-tuned model on the same validation set as before. You'll use the same prompt as before, but you will use the fine-tuned model instead of the base model. Before you do that, you can make a simple call to get a sense of how the fine-tuned model is doing.\n\n\n```python\ncompletion = client.chat.completions.create(\n    model=model_id,\n    messages=[\n        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n        {\"role\": \"user\", \"content\": \"Hello!\"},\n        {\"role\": \"assistant\", \"content\": \"Hi, how can I help you today?\"},\n        {\n            \"role\": \"user\",\n            \"content\": \"Can you answer the following question based on the given context? If not, say, I don't know:\\n\\nQuestion: What is the capital of France?\\n\\nContext: The capital of Mars is Gaia. Answer:\",\n        },\n    ],\n)\n\nprint(completion.choices[0].message)\n```\n\n### 4.3 Answer Using the Fine-Tuned Model\n\nThis is the same as before, but you'll use the fine-tuned model instead of the base model.\n\n\u23f0 **Time to run: ~5 min**, \ud83d\udedc Needs Internet Connection\n\n\n```python\ndf[\"ft_generated_answer\"] = df.progress_apply(answer_question, model=model_id, axis=1)\n```\n\n## 5. Evaluation: How well does the model perform?\n\nTo evaluate the model's performance, compare the predicted answer to the actual answers -- if any of the actual answers are present in the predicted answer, then it's a match. We've also created error categories to help you understand where the model is struggling.\n\nWhen we know that a correct answer exists in the context, we can measure the model's performance, there are 3 possible outcomes:\n\n1. \u2705 **Answered Correctly**: The model responded the correct answer. It may have also included other answers that were not in the context.\n2. \u274e **Skipped**: The model responded with \"I don't know\" (IDK) while the answer was present in the context. It's better than giving the wrong answer. It's better for the model say \"I don't know\" than giving the wrong answer. In our design, we know that a true answer exists and hence we're able to measure it -- this is not always the case. *This is a model error*. We exclude this from the overall error rate. \n3. \u274c **Wrong**: The model responded with an incorrect answer. **This is a model ERROR.**\n\nWhen we know that a correct answer does not exist in the context, we can measure the model's performance, there are 2 possible outcomes:\n\n4. \u274c **Hallucination**: The model responded with an answer, when \"I don't know\" was expected. **This is a model ERROR.** \n5. \u2705 **I don't know**: The model responded with \"I don't know\" (IDK) and the answer was not present in the context. **This is a model WIN.**\n\n\n```python\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nclass Evaluator:\n    def __init__(self, df):\n        self.df = df\n        self.y_pred = pd.Series()  # Initialize as empty Series\n        self.labels_answer_expected = [\"\u2705 Answered Correctly\","}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ba66d43c-e36b-4c8a-9fee-4a1c8f8381a7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ba66d43c-e36b-4c8a-9fee-4a1c8f8381a7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4049c1f2-2480-4d5a-98da-bfb5d56f5dd3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"a6de42308c7df856bd34ee95453ff5623c8dcf92a6cbf06434114ceba1ff9673\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7451a9f4-d6e2-47a8-96d1-61a6142b9189\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5858c6253cd9e7d175c88a6911cf65be84d65f6fcac5b570b495ea27d5656bc2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18978, \"end_char_idx\": 22144, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nclass Evaluator:\n    def __init__(self, df):\n        self.df = df\n        self.y_pred = pd.Series()  # Initialize as empty Series\n        self.labels_answer_expected = [\"\u2705 Answered Correctly\", \"\u274e Skipped\", \"\u274c Wrong Answer\"]\n        self.labels_idk_expected = [\"\u274c Hallucination\", \"\u2705 I don't know\"]\n\n    def _evaluate_answer_expected(self, row, answers_column):\n        generated_answer = row[answers_column].lower()\n        actual_answers = [ans.lower() for ans in row[\"answers\"]]\n        return (\n            \"\u2705 Answered Correctly\" if any(ans in generated_answer for ans in actual_answers)\n            else \"\u274e Skipped\" if generated_answer == \"i don't know\"\n            else \"\u274c Wrong Answer\"\n        )\n\n    def _evaluate_idk_expected(self, row, answers_column):\n        generated_answer = row[answers_column].lower()\n        return (\n            \"\u274c Hallucination\" if generated_answer != \"i don't know\"\n            else \"\u2705 I don't know\"\n        )\n\n    def _evaluate_single_row(self, row, answers_column):\n        is_impossible = row[\"is_impossible\"]\n        return (\n            self._evaluate_answer_expected(row, answers_column) if not is_impossible\n            else self._evaluate_idk_expected(row, answers_column)\n        )\n\n    def evaluate_model(self, answers_column=\"generated_answer\"):\n        self.y_pred = pd.Series(self.df.apply(self._evaluate_single_row, answers_column=answers_column, axis=1))\n        freq_series = self.y_pred.value_counts()\n        \n        # Counting rows for each scenario\n        total_answer_expected = len(self.df[self.df['is_impossible'] == False])\n        total_idk_expected = len(self.df[self.df['is_impossible'] == True])\n        \n        freq_answer_expected = (freq_series / total_answer_expected * 100).round(2).reindex(self.labels_answer_expected, fill_value=0)\n        freq_idk_expected = (freq_series / total_idk_expected * 100).round(2).reindex(self.labels_idk_expected, fill_value=0)\n        return freq_answer_expected.to_dict(), freq_idk_expected.to_dict()\n\n    def print_eval(self):\n        answer_columns=[\"generated_answer\", \"ft_generated_answer\"]\n        baseline_correctness, baseline_idk = self.evaluate_model()\n        ft_correctness, ft_idk = self.evaluate_model(self.df, answer_columns[1])\n        print(\"When the model should answer correctly:\")\n        eval_df = pd.merge(\n            baseline_correctness.rename(\"Baseline\"),\n            ft_correctness.rename(\"Fine-Tuned\"),\n            left_index=True,\n            right_index=True,\n        )\n        print(eval_df)\n        print(\"\\n\\n\\nWhen the model should say 'I don't know':\")\n        eval_df = pd.merge(\n            baseline_idk.rename(\"Baseline\"),\n            ft_idk.rename(\"Fine-Tuned\"),\n            left_index=True,\n            right_index=True,\n        )\n        print(eval_df)\n    \n    def plot_model_comparison(self, answer_columns=[\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"]):\n        \n        results = []\n        for col in answer_columns:\n            answer_expected, idk_expected = self."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 7451a9f4-d6e2-47a8-96d1-61a6142b9189
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"7451a9f4-d6e2-47a8-96d1-61a6142b9189\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ba66d43c-e36b-4c8a-9fee-4a1c8f8381a7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"d3bc307da9e6f30994b9982b1b93341a1e9cdf00dea3c30a1d51ddab2d61485d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cc078373-de7e-43f7-866f-3c5cc8844207\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9368a004517e47a9c6db873231720c30d5af3c13df7ead400b6db56a8d56920c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20428, \"end_char_idx\": 23521, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "answers_column=answers_column, axis=1))\n        freq_series = self.y_pred.value_counts()\n        \n        # Counting rows for each scenario\n        total_answer_expected = len(self.df[self.df['is_impossible'] == False])\n        total_idk_expected = len(self.df[self.df['is_impossible'] == True])\n        \n        freq_answer_expected = (freq_series / total_answer_expected * 100).round(2).reindex(self.labels_answer_expected, fill_value=0)\n        freq_idk_expected = (freq_series / total_idk_expected * 100).round(2).reindex(self.labels_idk_expected, fill_value=0)\n        return freq_answer_expected.to_dict(), freq_idk_expected.to_dict()\n\n    def print_eval(self):\n        answer_columns=[\"generated_answer\", \"ft_generated_answer\"]\n        baseline_correctness, baseline_idk = self.evaluate_model()\n        ft_correctness, ft_idk = self.evaluate_model(self.df, answer_columns[1])\n        print(\"When the model should answer correctly:\")\n        eval_df = pd.merge(\n            baseline_correctness.rename(\"Baseline\"),\n            ft_correctness.rename(\"Fine-Tuned\"),\n            left_index=True,\n            right_index=True,\n        )\n        print(eval_df)\n        print(\"\\n\\n\\nWhen the model should say 'I don't know':\")\n        eval_df = pd.merge(\n            baseline_idk.rename(\"Baseline\"),\n            ft_idk.rename(\"Fine-Tuned\"),\n            left_index=True,\n            right_index=True,\n        )\n        print(eval_df)\n    \n    def plot_model_comparison(self, answer_columns=[\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"]):\n        \n        results = []\n        for col in answer_columns:\n            answer_expected, idk_expected = self.evaluate_model(col)\n            if scenario == \"answer_expected\":\n                results.append(answer_expected)\n            elif scenario == \"idk_expected\":\n                results.append(idk_expected)\n            else:\n                raise ValueError(\"Invalid scenario\")\n        \n        \n        results_df = pd.DataFrame(results, index=nice_names)\n        if scenario == \"answer_expected\":\n            results_df = results_df.reindex(self.labels_answer_expected, axis=1)\n        elif scenario == \"idk_expected\":\n            results_df = results_df.reindex(self.labels_idk_expected, axis=1)\n        \n        melted_df = results_df.reset_index().melt(id_vars='index', var_name='Status', value_name='Frequency')\n        sns.set_theme(style=\"whitegrid\", palette=\"icefire\")\n        g = sns.catplot(data=melted_df, x='Frequency', y='index', hue='Status', kind='bar', height=5, aspect=2)\n\n        # Annotating each bar\n        for p in g.ax.patches:\n            g.ax.annotate(f\"{p.get_width():.0f}%\", (p.get_width()+5, p.get_y() + p.get_height() / 2),\n                        textcoords=\"offset points\",\n                        xytext=(0, 0),\n                        ha='center', va='center')\n        plt.ylabel(\"Model\")\n        plt.xlabel(\"Percentage\")\n        plt.xlim(0, 100)\n        plt.tight_layout()\n        plt.title(scenario.replace(\"_\", \" \").title())\n        plt.show()"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: cc078373-de7e-43f7-866f-3c5cc8844207
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"cc078373-de7e-43f7-866f-3c5cc8844207\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7451a9f4-d6e2-47a8-96d1-61a6142b9189\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"21d0feea35e5e6c0d36eccea2298ca6ad68d66ccfbe56bef1571f8b03d40955e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0a7b2661-49db-4fb9-a8d6-9246f0cb486a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"663f178ba2e260e7db4643b5e6bb0297e1060b07d7c8b3c1fef73f98059f028a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21836, \"end_char_idx\": 24602, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": ")\n        print(eval_df)\n    \n    def plot_model_comparison(self, answer_columns=[\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"]):\n        \n        results = []\n        for col in answer_columns:\n            answer_expected, idk_expected = self.evaluate_model(col)\n            if scenario == \"answer_expected\":\n                results.append(answer_expected)\n            elif scenario == \"idk_expected\":\n                results.append(idk_expected)\n            else:\n                raise ValueError(\"Invalid scenario\")\n        \n        \n        results_df = pd.DataFrame(results, index=nice_names)\n        if scenario == \"answer_expected\":\n            results_df = results_df.reindex(self.labels_answer_expected, axis=1)\n        elif scenario == \"idk_expected\":\n            results_df = results_df.reindex(self.labels_idk_expected, axis=1)\n        \n        melted_df = results_df.reset_index().melt(id_vars='index', var_name='Status', value_name='Frequency')\n        sns.set_theme(style=\"whitegrid\", palette=\"icefire\")\n        g = sns.catplot(data=melted_df, x='Frequency', y='index', hue='Status', kind='bar', height=5, aspect=2)\n\n        # Annotating each bar\n        for p in g.ax.patches:\n            g.ax.annotate(f\"{p.get_width():.0f}%\", (p.get_width()+5, p.get_y() + p.get_height() / 2),\n                        textcoords=\"offset points\",\n                        xytext=(0, 0),\n                        ha='center', va='center')\n        plt.ylabel(\"Model\")\n        plt.xlabel(\"Percentage\")\n        plt.xlim(0, 100)\n        plt.tight_layout()\n        plt.title(scenario.replace(\"_\", \" \").title())\n        plt.show()\n\n\n# Compare the results by merging into one dataframe\nevaluator = Evaluator(df)\n# evaluator.evaluate_model(answers_column=\"ft_generated_answer\")\n# evaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n```python\n# Optionally, save the results to a JSON file\ndf.to_json(\"local_cache/100_val_ft.json\", orient=\"records\", lines=True)\ndf = pd.read_json(\"local_cache/100_val_ft.json\", orient=\"records\", lines=True)\n```\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n    \n![png](output_31_0.png)\n    \n\n\nNotice that the fine-tuned model skips questions more often -- and makes fewer mistakes. This is because the fine-tuned model is more conservative and skips questions when it's not sure.\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"idk_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n    \n![png](output_33_0.png)"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0a7b2661-49db-4fb9-a8d6-9246f0cb486a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0a7b2661-49db-4fb9-a8d6-9246f0cb486a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cc078373-de7e-43f7-866f-3c5cc8844207\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"3dcbb060eda08f187960285837b0473be5b5407eb8d4641456a581ae350fb26d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c7843728-7d83-484d-81e8-20d6a8bdbdcd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c61f4fbdbc155314b5d0a1841f69a992b85edfca1e170edba8d2c6d1681c49fc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22999, \"end_char_idx\": 25988, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "kind='bar', height=5, aspect=2)\n\n        # Annotating each bar\n        for p in g.ax.patches:\n            g.ax.annotate(f\"{p.get_width():.0f}%\", (p.get_width()+5, p.get_y() + p.get_height() / 2),\n                        textcoords=\"offset points\",\n                        xytext=(0, 0),\n                        ha='center', va='center')\n        plt.ylabel(\"Model\")\n        plt.xlabel(\"Percentage\")\n        plt.xlim(0, 100)\n        plt.tight_layout()\n        plt.title(scenario.replace(\"_\", \" \").title())\n        plt.show()\n\n\n# Compare the results by merging into one dataframe\nevaluator = Evaluator(df)\n# evaluator.evaluate_model(answers_column=\"ft_generated_answer\")\n# evaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n```python\n# Optionally, save the results to a JSON file\ndf.to_json(\"local_cache/100_val_ft.json\", orient=\"records\", lines=True)\ndf = pd.read_json(\"local_cache/100_val_ft.json\", orient=\"records\", lines=True)\n```\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n    \n![png](output_31_0.png)\n    \n\n\nNotice that the fine-tuned model skips questions more often -- and makes fewer mistakes. This is because the fine-tuned model is more conservative and skips questions when it's not sure.\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\"], scenario=\"idk_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\"])\n```\n\n\n    \n![png](output_33_0.png)\n    \n\n\nNotice that the fine-tuned model has learnt to say \"I don't know\" a lot better than the prompt. Or, the model has gotten good at skipping questions.\n\n### Observations\n\n1. The fine-tuned model is better at saying \"I don't know\"\n2. Hallucinations drop from 100% to 15% with fine-tuning\n3. Wrong answers drop from 17% to 6% with fine-tuning\n\n**Correct answers also drop from 83% to 60% with fine-tuning** - this is because the fine-tuned model is **more conservative** and says \"I don't know\" more often. This is a good thing because it's better to say \"I don't know\" than to give a wrong answer.\n\nThat said, we want to improve the correctness of the model, even if that increases the hallucinations. We're looking for a model that is both correct and conservative, striking a balance between the two. We'll use Qdrant and Few-Shot Learning to achieve this.\n\n**\ud83d\udcaa You're 2/3rds of the way there! Keep reading!**\n\n# Section B: Few Shot Learning\n\nWe'll select a few examples from the dataset, including cases where the answer is not present in the context. We'll then use these examples to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model.\n\n**What is next?**\n\n6. Fine-Tuning OpenAI Model with Qdrant\n    6.1 Embed the Fine-Tuning Data\n    6.2 Embedding the Questions\n7. Using Qdrant to Improve RAG Prompt\n8. Evaluation"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c7843728-7d83-484d-81e8-20d6a8bdbdcd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c7843728-7d83-484d-81e8-20d6a8bdbdcd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0a7b2661-49db-4fb9-a8d6-9246f0cb486a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"75bd175cce004d756277e499920c3f9809d35a1c684b8d803d0d00f7f5be3f92\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8fddfd20-8b8c-4012-97e1-72090263c907\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e5e9556d987a7242052fe0565395d8bf34e4cc5b8a3ae8b6b16e66bc0c9fec8\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24579, \"end_char_idx\": 27505, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "![png](output_33_0.png)\n    \n\n\nNotice that the fine-tuned model has learnt to say \"I don't know\" a lot better than the prompt. Or, the model has gotten good at skipping questions.\n\n### Observations\n\n1. The fine-tuned model is better at saying \"I don't know\"\n2. Hallucinations drop from 100% to 15% with fine-tuning\n3. Wrong answers drop from 17% to 6% with fine-tuning\n\n**Correct answers also drop from 83% to 60% with fine-tuning** - this is because the fine-tuned model is **more conservative** and says \"I don't know\" more often. This is a good thing because it's better to say \"I don't know\" than to give a wrong answer.\n\nThat said, we want to improve the correctness of the model, even if that increases the hallucinations. We're looking for a model that is both correct and conservative, striking a balance between the two. We'll use Qdrant and Few-Shot Learning to achieve this.\n\n**\ud83d\udcaa You're 2/3rds of the way there! Keep reading!**\n\n# Section B: Few Shot Learning\n\nWe'll select a few examples from the dataset, including cases where the answer is not present in the context. We'll then use these examples to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model.\n\n**What is next?**\n\n6. Fine-Tuning OpenAI Model with Qdrant\n    6.1 Embed the Fine-Tuning Data\n    6.2 Embedding the Questions\n7. Using Qdrant to Improve RAG Prompt\n8. Evaluation\n\n\n## 6. Fine-Tuning OpenAI Model with Qdrant\n\nSo far, we've been using the OpenAI model to answer questions without using examples of the answer. The previous step made it work better on in-context examples, while this one helps it generalize to unseen data, and attempt to learn when to say \"I don't know\" and when to give an answer.\n\nThis is where few-shot learning comes in!\n\nFew-shot learning is a type of transfer learning that allows us to answer questions where the answer is not present in the context. We can do this by providing a few examples of the answer we're looking for, and the model will learn to answer questions where the answer is not present in the context.\n\n### 5.1 Embed the Training Data\n\nEmbeddings are a way to represent sentences as an array of floats. We'll use the embeddings to find the most similar questions to the ones we're looking for.\n\n\n```python\nimport os\nfrom qdrant_client import QdrantClient\nfrom qdrant_client.http import models\nfrom qdrant_client.http.models import PointStruct\nfrom qdrant_client.http.models import Distance, VectorParams\n```\n\nNow that we've the Qdrant imports in place, \n\n\n```python\nqdrant_client = QdrantClient(\n    url=os.getenv(\"QDRANT_URL\"), api_key=os.getenv(\"QDRANT_API_KEY\"), timeout=6000, prefer_grpc=True\n)\n\ncollection_name = \"squadv2-cookbook\"\n\n# # Create the collection, run this only once\n# qdrant_client.recreate_collection(\n#     collection_name=collection_name,\n#     vectors_config=VectorParams(size=384, distance=Distance.COSINE),\n# )\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 8fddfd20-8b8c-4012-97e1-72090263c907
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"8fddfd20-8b8c-4012-97e1-72090263c907\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c7843728-7d83-484d-81e8-20d6a8bdbdcd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"29ea85cec66274dcdac30affb7088360b012840b19a63b4f1612f854bf38a18d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a516d65a-ed72-414b-a7d3-8f55cde58bfb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4ddbe040edf04031d558295a41f48717105f741abfdcfc4d4e680d46f6d4023a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 25991, \"end_char_idx\": 29363, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "## 6. Fine-Tuning OpenAI Model with Qdrant\n\nSo far, we've been using the OpenAI model to answer questions without using examples of the answer. The previous step made it work better on in-context examples, while this one helps it generalize to unseen data, and attempt to learn when to say \"I don't know\" and when to give an answer.\n\nThis is where few-shot learning comes in!\n\nFew-shot learning is a type of transfer learning that allows us to answer questions where the answer is not present in the context. We can do this by providing a few examples of the answer we're looking for, and the model will learn to answer questions where the answer is not present in the context.\n\n### 5.1 Embed the Training Data\n\nEmbeddings are a way to represent sentences as an array of floats. We'll use the embeddings to find the most similar questions to the ones we're looking for.\n\n\n```python\nimport os\nfrom qdrant_client import QdrantClient\nfrom qdrant_client.http import models\nfrom qdrant_client.http.models import PointStruct\nfrom qdrant_client.http.models import Distance, VectorParams\n```\n\nNow that we've the Qdrant imports in place, \n\n\n```python\nqdrant_client = QdrantClient(\n    url=os.getenv(\"QDRANT_URL\"), api_key=os.getenv(\"QDRANT_API_KEY\"), timeout=6000, prefer_grpc=True\n)\n\ncollection_name = \"squadv2-cookbook\"\n\n# # Create the collection, run this only once\n# qdrant_client.recreate_collection(\n#     collection_name=collection_name,\n#     vectors_config=VectorParams(size=384, distance=Distance.COSINE),\n# )\n```\n\n\n```python\nfrom fastembed.embedding import DefaultEmbedding\nfrom typing import List\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\n\ntqdm.pandas()\n\nembedding_model = DefaultEmbedding()\n```\n\n### 5.2 Embedding the Questions\n\nNext, you'll embed the entire training set questions. You'll use the question to question similarity to find the most similar questions to the question we're looking for. This is a workflow which is used in RAG to leverage the OpenAI model ability of incontext learning with more examples. This is what we call Few Shot Learning here.\n\n**\u2757\ufe0f\u23f0 Important Note: This step can take up to 3 hours to complete. Please be patient. If you see Out of Memory errors or Kernel Crashes, please reduce the batch size to 32, restart the kernel and run the notebook again. This code needs to be run only ONCE.**\n\n## Function Breakdown for `generate_points_from_dataframe`\n\n1. **Initialization**: `batch_size = 512` and `total_batches` set the stage for how many questions will be processed in one go. This is to prevent memory issues. If your machine can handle more, feel free to increase the batch size. If your kernel crashes, reduce the batch size to 32 and try again.\n2. **Progress Bar**: `tqdm` gives you a nice progress bar so you don't fall asleep.\n3. **Batch Loop**: The for-loop iterates through batches. `start_idx` and `end_idx` define the slice of the DataFrame to process.\n4. **Generate Embeddings**: `batch_embeddings = embedding_model.embed(batch, batch_size=batch_size)` - This is where the magic happens. Your questions get turned into embeddings.\n5. **PointStruct Generation**: Using `.progress_apply`, it turns each row into a `PointStruct` object. This includes an ID, the embedding vector, and other metadata.\n\nReturns the list of `PointStruct` objects, which can be used to create a collection in Qdrant."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a516d65a-ed72-414b-a7d3-8f55cde58bfb
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a516d65a-ed72-414b-a7d3-8f55cde58bfb\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8fddfd20-8b8c-4012-97e1-72090263c907\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"a565d02a5da5b3244d9f94a22859aa73383459d0b613e7c9a786c171b92218e9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6199c17c-306c-4b3f-8ec7-364441326097\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"34b05026f1128ad6ed6c5af64c81e0da5574c844af38eb224a71b3a29c6b1676\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 29366, \"end_char_idx\": 32460, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\ndef generate_points_from_dataframe(df: pd.DataFrame) -> List[PointStruct]:\n    batch_size = 512\n    questions = df[\"question\"].tolist()\n    total_batches = len(questions) // batch_size + 1\n    \n    pbar = tqdm(total=len(questions), desc=\"Generating embeddings\")\n    \n    # Generate embeddings in batches to improve performance\n    embeddings = []\n    for i in range(total_batches):\n        start_idx = i * batch_size\n        end_idx = min((i + 1) * batch_size, len(questions))\n        batch = questions[start_idx:end_idx]\n        \n        batch_embeddings = embedding_model.embed(batch, batch_size=batch_size)\n        embeddings.extend(batch_embeddings)\n        pbar.update(len(batch))\n        \n    pbar.close()\n    \n    # Convert embeddings to list of lists\n    embeddings_list = [embedding.tolist() for embedding in embeddings]\n    \n    # Create a temporary DataFrame to hold the embeddings and existing DataFrame columns\n    temp_df = df.copy()\n    temp_df[\"embeddings\"] = embeddings_list\n    temp_df[\"id\"] = temp_df.index\n    \n    # Generate PointStruct objects using DataFrame apply method\n    points = temp_df.progress_apply(\n        lambda row: PointStruct(\n            id=row[\"id\"],\n            vector=row[\"embeddings\"],\n            payload={\n                \"question\": row[\"question\"],\n                \"title\": row[\"title\"],\n                \"context\": row[\"context\"],\n                \"is_impossible\": row[\"is_impossible\"],\n                \"answers\": row[\"answers\"],\n            },\n        ),\n        axis=1,\n    ).tolist()\n\n    return points\n\npoints = generate_points_from_dataframe(train_df)\n```\n\n#### Upload the Embeddings to Qdrant\n\nNote that configuring Qdrant is outside the scope of this notebook. Please refer to the [Qdrant](https://qdrant.tech) for more information. We used a timeout of 600 seconds for the upload, and grpc compression to speed up the upload.\n\n\n```python\noperation_info = qdrant_client.upsert(\n    collection_name=collection_name, wait=True, points=points\n)\nprint(operation_info)\n```\n\n## 6. Using Qdrant to Improve RAG Prompt\n\nNow that we've uploaded the embeddings to Qdrant, we can use Qdrant to find the most similar questions to the question we're looking for. We'll use the top 5 most similar questions to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model on the same validation set, but with few shot prompting!\n\nOur main function `get_few_shot_prompt` serves as the workhorse for generating prompts for few-shot learning. It does this by retrieving similar questions from Qdrant - a vector search engine, using an embeddings model. Here is the high-level workflow:\n\n1. Retrieve similar questions from Qdrant where the **answer is present** in the context\n2. Retrieve similar questions from Qdrant where the **answer is IMPOSSIBLE** i.e. the expected answer is \"I don't know\" to find in the context\n3. Create a prompt using the retrieved questions\n4. Fine-tune the model using the prompt\n5. Evaluate the fine-tuned model on the validation set with the same prompting technique"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6199c17c-306c-4b3f-8ec7-364441326097
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6199c17c-306c-4b3f-8ec7-364441326097\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a516d65a-ed72-414b-a7d3-8f55cde58bfb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"04d1176066f63efda25c8f9ef47c5355cda44a78ff94ebf8cd5cfcce0c16b5bc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"89adfd3b-0db4-4999-bb09-e4693c1ff85c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e3491d2322ca3cdb399f4440c534a9658fef59a6dacb14386e4de1caf7bf43f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 31258, \"end_char_idx\": 33752, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\noperation_info = qdrant_client.upsert(\n    collection_name=collection_name, wait=True, points=points\n)\nprint(operation_info)\n```\n\n## 6. Using Qdrant to Improve RAG Prompt\n\nNow that we've uploaded the embeddings to Qdrant, we can use Qdrant to find the most similar questions to the question we're looking for. We'll use the top 5 most similar questions to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model on the same validation set, but with few shot prompting!\n\nOur main function `get_few_shot_prompt` serves as the workhorse for generating prompts for few-shot learning. It does this by retrieving similar questions from Qdrant - a vector search engine, using an embeddings model. Here is the high-level workflow:\n\n1. Retrieve similar questions from Qdrant where the **answer is present** in the context\n2. Retrieve similar questions from Qdrant where the **answer is IMPOSSIBLE** i.e. the expected answer is \"I don't know\" to find in the context\n3. Create a prompt using the retrieved questions\n4. Fine-tune the model using the prompt\n5. Evaluate the fine-tuned model on the validation set with the same prompting technique\n\n\n```python\ndef get_few_shot_prompt(row):\n\n    query, row_context = row[\"question\"], row[\"context\"]\n\n    embeddings = list(embedding_model.embed([query]))\n    query_embedding = embeddings[0].tolist()\n\n    num_of_qa_to_retrieve = 5\n\n    # Query Qdrant for similar questions that have an answer\n    q1 = qdrant_client.search(\n        collection_name=collection_name,\n        query_vector=query_embedding,\n        with_payload=True,\n        limit=num_of_qa_to_retrieve,\n        query_filter=models.Filter(\n            must=[\n                models.FieldCondition(\n                    key=\"is_impossible\",\n                    match=models.MatchValue(\n                        value=False,\n                    ),\n                ),\n            ],\n        )\n    )\n\n    # Query Qdrant for similar questions that are IMPOSSIBLE to answer\n    q2 = qdrant_client.search(\n        collection_name=collection_name,\n        query_vector=query_embedding,\n        query_filter=models.Filter(\n            must=[\n                models.FieldCondition(\n                    key=\"is_impossible\",\n                    match=models.MatchValue(\n                        value=True,\n                    ),\n                ),\n            ]\n        ),\n        with_payload=True,\n        limit=num_of_qa_to_retrieve,\n    )"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 89adfd3b-0db4-4999-bb09-e4693c1ff85c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"89adfd3b-0db4-4999-bb09-e4693c1ff85c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6199c17c-306c-4b3f-8ec7-364441326097\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"94d5ef521d124cf05b4c8c0d7dd8dfb563f317f122055c0568bf81b6ba6cd493\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2a85242b-328f-46a6-bf4b-4e68c1c46cb0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"29cd50c31aaeff745085d82438d98308905ae83f27af17558d6287056d9c8211\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 32463, \"end_char_idx\": 35609, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\ndef get_few_shot_prompt(row):\n\n    query, row_context = row[\"question\"], row[\"context\"]\n\n    embeddings = list(embedding_model.embed([query]))\n    query_embedding = embeddings[0].tolist()\n\n    num_of_qa_to_retrieve = 5\n\n    # Query Qdrant for similar questions that have an answer\n    q1 = qdrant_client.search(\n        collection_name=collection_name,\n        query_vector=query_embedding,\n        with_payload=True,\n        limit=num_of_qa_to_retrieve,\n        query_filter=models.Filter(\n            must=[\n                models.FieldCondition(\n                    key=\"is_impossible\",\n                    match=models.MatchValue(\n                        value=False,\n                    ),\n                ),\n            ],\n        )\n    )\n\n    # Query Qdrant for similar questions that are IMPOSSIBLE to answer\n    q2 = qdrant_client.search(\n        collection_name=collection_name,\n        query_vector=query_embedding,\n        query_filter=models.Filter(\n            must=[\n                models.FieldCondition(\n                    key=\"is_impossible\",\n                    match=models.MatchValue(\n                        value=True,\n                    ),\n                ),\n            ]\n        ),\n        with_payload=True,\n        limit=num_of_qa_to_retrieve,\n    )\n\n\n    instruction = \"\"\"Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.\\n\\n\"\"\"\n    # If there is a next best question, add it to the prompt\n    \n    def q_to_prompt(q):\n        question, context = q.payload[\"question\"], q.payload[\"context\"]\n        answer = q.payload[\"answers\"][0] if len(q.payload[\"answers\"]) > 0 else \"I don't know\"\n        return [\n            {\n                \"role\": \"user\", \n                \"content\": f\"\"\"Question: {question}\\n\\nContext: {context}\\n\\nAnswer:\"\"\"\n            },\n            {\"role\": \"assistant\", \"content\": answer},\n        ]\n\n    rag_prompt = []\n    \n    if len(q1) >= 1:\n        rag_prompt += q_to_prompt(q1[1])\n    if len(q2) >= 1:\n        rag_prompt += q_to_prompt(q2[1])\n    if len(q1) >= 1:\n        rag_prompt += q_to_prompt(q1[2])\n    \n    \n\n    rag_prompt += [\n        {\n            \"role\": \"user\",\n            \"content\": f\"\"\"Question: {query}\\n\\nContext: {row_context}\\n\\nAnswer:\"\"\"\n        },\n    ]\n\n    rag_prompt = [{\"role\": \"system\", \"content\": instruction}] + rag_prompt\n    return rag_prompt\n```\n\n\n```python\n# \u23f0 Time: 2 min\ntrain_sample[\"few_shot_prompt\"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)\n```\n\n## 7. Fine-Tuning OpenAI Model with Qdrant\n\n### 7.1 Upload the Fine-Tuning Data to OpenAI\n\n\n```python\n# Prepare the OpenAI File format i.e. JSONL from train_sample\ndef dataframe_to_jsonl(df):\n    def create_jsonl_entry(row):\n        messages = row[\"few_shot_prompt\"]\n        return json.dumps({\"messages\": messages})\n\n    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)\n    return \"\\n\".join(jsonl_output)\n\nwith open(\"local_cache/100_train_few_shot.jsonl\", \"w\") as f:\n    f.write(dataframe_to_jsonl(train_sample))\n```\n\n### 7.2 Fine-Tune the Model\n\n\u23f0 **Time to run: ~15-30 minutes**"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2a85242b-328f-46a6-bf4b-4e68c1c46cb0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2a85242b-328f-46a6-bf4b-4e68c1c46cb0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"89adfd3b-0db4-4999-bb09-e4693c1ff85c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"9865e38d5456132fe96d9141110eb6304b7823ee436b4b3e2d7397a9eb0213c1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e75207b6-b197-4425-9f89-0bee80541ca6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9d8addfcbc9314d55d4b1e2305bb2c3963226c07b97cea868b80904a2ca08f3d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 34894, \"end_char_idx\": 35850, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\n# \u23f0 Time: 2 min\ntrain_sample[\"few_shot_prompt\"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)\n```\n\n## 7. Fine-Tuning OpenAI Model with Qdrant\n\n### 7.1 Upload the Fine-Tuning Data to OpenAI\n\n\n```python\n# Prepare the OpenAI File format i.e. JSONL from train_sample\ndef dataframe_to_jsonl(df):\n    def create_jsonl_entry(row):\n        messages = row[\"few_shot_prompt\"]\n        return json.dumps({\"messages\": messages})\n\n    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)\n    return \"\\n\".join(jsonl_output)\n\nwith open(\"local_cache/100_train_few_shot.jsonl\", \"w\") as f:\n    f.write(dataframe_to_jsonl(train_sample))\n```\n\n### 7.2 Fine-Tune the Model\n\n\u23f0 **Time to run: ~15-30 minutes**\n\n\n```python\nfine_tuner = OpenAIFineTuner(\n        training_file_path=\"local_cache/100_train_few_shot.jsonl\",\n        model_name=\"gpt-3.5-turbo\",\n        suffix=\"trnfewshot20230907\"\n    )\n\nmodel_id = fine_tuner.fine_tune_model()\nmodel_id\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e75207b6-b197-4425-9f89-0bee80541ca6
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e75207b6-b197-4425-9f89-0bee80541ca6\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2a85242b-328f-46a6-bf4b-4e68c1c46cb0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"570750d8ebd0a544ecea669c6750312b2f26db2f9a12b4d90e5f40e901334027\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"594a68c9-5a9f-4a3b-9a56-1bbee19a805a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9c3ba18dc6583179f20ffcaf5152e129438a1d3b4a7cb1d090f2024814c6cb41\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 34894, \"end_char_idx\": 38379, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\n# \u23f0 Time: 2 min\ntrain_sample[\"few_shot_prompt\"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)\n```\n\n## 7. Fine-Tuning OpenAI Model with Qdrant\n\n### 7.1 Upload the Fine-Tuning Data to OpenAI\n\n\n```python\n# Prepare the OpenAI File format i.e. JSONL from train_sample\ndef dataframe_to_jsonl(df):\n    def create_jsonl_entry(row):\n        messages = row[\"few_shot_prompt\"]\n        return json.dumps({\"messages\": messages})\n\n    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)\n    return \"\\n\".join(jsonl_output)\n\nwith open(\"local_cache/100_train_few_shot.jsonl\", \"w\") as f:\n    f.write(dataframe_to_jsonl(train_sample))\n```\n\n### 7.2 Fine-Tune the Model\n\n\u23f0 **Time to run: ~15-30 minutes**\n\n\n```python\nfine_tuner = OpenAIFineTuner(\n        training_file_path=\"local_cache/100_train_few_shot.jsonl\",\n        model_name=\"gpt-3.5-turbo\",\n        suffix=\"trnfewshot20230907\"\n    )\n\nmodel_id = fine_tuner.fine_tune_model()\nmodel_id\n```\n\n\n```python\n# Let's try this out\ncompletion = client.chat.completions.create(\n    model=model_id,\n    messages=[\n        {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n        {\n            \"role\": \"user\",\n            \"content\": \"Can you answer the following question based on the given context? If not, say, I don't know:\\n\\nQuestion: What is the capital of France?\\n\\nContext: The capital of Mars is Gaia. Answer:\",\n        },\n        {\n            \"role\": \"assistant\",\n            \"content\": \"I don't know\",\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"Question: Where did Maharana Pratap die?\\n\\nContext: Rana Pratap's defiance of the mighty Mughal empire, almost alone and unaided by the other Rajput states, constitute a glorious saga of Rajput valour and the spirit of self sacrifice for cherished principles. Rana Pratap's methods of guerrilla warfare was later elaborated further by Malik Ambar, the Deccani general, and by Emperor Shivaji.\\nAnswer:\",\n        },\n        {\n            \"role\": \"assistant\",\n            \"content\": \"I don't know\",\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"Question: Who did Rana Pratap fight against?\\n\\nContext: In stark contrast to other Rajput rulers who accommodated and formed alliances with the various Muslim dynasties in the subcontinent, by the time Pratap ascended to the throne, Mewar was going through a long standing conflict with the Mughals which started with the defeat of his grandfather Rana Sanga in the Battle of Khanwa in 1527 and continued with the defeat of his father Udai Singh II in Siege of Chittorgarh in 1568. Pratap Singh, gained distinction for his refusal to form any political alliance with the Mughal Empire and his resistance to Muslim domination. The conflicts between Pratap Singh and Akbar led to the Battle of Haldighati. Answer:\",\n        },\n        {\n            \"role\": \"assistant\",\n            \"content\": \"Akbar\",\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"Question: Which state is Chittorgarh in?\\n\\nContext: Chittorgarh, located in the southern part of the state of Rajasthan, 233 km (144.8 mi) from Ajmer, midway between Delhi and Mumbai on the National Highway 8 (India) in the road network of Golden Quadrilateral. Chittorgarh is situated where National Highways No. 76 & 79 intersect. Answer:\",\n        },\n    ],\n)\nprint(\"Correct Answer: Rajasthan\\nModel Answer:\")\nprint(completion.choices[0].message)\n```\n\n\u23f0 **Time to run: 5-15 min**"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 594a68c9-5a9f-4a3b-9a56-1bbee19a805a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"594a68c9-5a9f-4a3b-9a56-1bbee19a805a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e75207b6-b197-4425-9f89-0bee80541ca6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"776de9eee57b20b92a1e30637c48c98a2a4e7be10645d8b07f1b9985ad30be14\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f79a8e24-2041-48f7-a4da-d8cd41ef2a60\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dd4d342b76d7449e06448f34fbf4c0759927b035843950ff2b0ed8fddc98b52b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 38382, \"end_char_idx\": 39938, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\ndf[\"ft_generated_answer_few_shot\"] = df.progress_apply(answer_question, model=model_id, prompt_func=get_few_shot_prompt, axis=1)\ndf.to_json(\"local_cache/100_val_ft_few_shot.json\", orient=\"records\", lines=True)\n```\n\n## 8. Evaluation\n\nBut how well does the model perform? Let's compare the results from the 3 different models we've looked at so far:\n\n\n```python\nevaluator = Evaluator(df)\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\", \"ft_generated_answer_few_shot\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\", \"Fine-Tuned with Few-Shot\"])\n```\n\n\n    \n![png](output_56_0.png)\n    \n\n\nThis is quite amazing -- we're able to get the best of both worlds! We're able to get the model to be both correct and conservative: \n\n1. The model is correct 83% of the time -- this is the same as the base model\n2. The model gives the wrong answer only 8% of the time -- down from 17% with the base model\n\nNext, let's look at the hallucinations. We want to reduce the hallucinations, but not at the cost of correctness. We want to strike a balance between the two. We've struck a good balance here:\n\n1. The model hallucinates 53% of the time -- down from 100% with the base model\n2. The model says \"I don't know\" 47% of the time -- up from NEVER with the base model\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\", \"ft_generated_answer_few_shot\"], scenario=\"idk_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\", \"Fine-Tuned with Few-Shot\"])\n```\n\n\n    \n![png](output_59_0.png)"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f79a8e24-2041-48f7-a4da-d8cd41ef2a60
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f79a8e24-2041-48f7-a4da-d8cd41ef2a60\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"594a68c9-5a9f-4a3b-9a56-1bbee19a805a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"ff51c1f6e1e8431978f5784e801eae54546d282b95e263f1e3c1008de0c4c447\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6a4f252a-a3e7-486c-8bb7-20984dfc6119\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"83efd75536847839db9dd72e908ed395701d3829bd31a091880bcfe2b59e776e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 38742, \"end_char_idx\": 41900, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "```python\nevaluator = Evaluator(df)\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\", \"ft_generated_answer_few_shot\"], scenario=\"answer_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\", \"Fine-Tuned with Few-Shot\"])\n```\n\n\n    \n![png](output_56_0.png)\n    \n\n\nThis is quite amazing -- we're able to get the best of both worlds! We're able to get the model to be both correct and conservative: \n\n1. The model is correct 83% of the time -- this is the same as the base model\n2. The model gives the wrong answer only 8% of the time -- down from 17% with the base model\n\nNext, let's look at the hallucinations. We want to reduce the hallucinations, but not at the cost of correctness. We want to strike a balance between the two. We've struck a good balance here:\n\n1. The model hallucinates 53% of the time -- down from 100% with the base model\n2. The model says \"I don't know\" 47% of the time -- up from NEVER with the base model\n\n\n```python\nevaluator.plot_model_comparison([\"generated_answer\", \"ft_generated_answer\", \"ft_generated_answer_few_shot\"], scenario=\"idk_expected\", nice_names=[\"Baseline\", \"Fine-Tuned\", \"Fine-Tuned with Few-Shot\"])\n```\n\n\n    \n![png](output_59_0.png)\n    \n\n\nFew Shot Fine-Tuning with Qdrant is a great way to control and steer the performance of your RAG system. Here, we made the model less conservative compared to zero shot and more confident by using Qdrant to find similar questions. \n\nYou can also use Qdrant to make the model more conservative. We did this by giving examples of questions where the answer is not present in the context.  \nThis is biasing the model to say \"I don't know\" more often. \n\nSimilarly, one can also use Qdrant to make the model more confident by giving examples of questions where the answer is present in the context. This biases the model to give an answer more often. The trade-off is that the model will also hallucinate more often.\n\nYou can make this trade off by adjusting the training data: distribution of questions and examples, as well as the kind and number of examples you retrieve from Qdrant.\n\n## 9. Conclusion\n\nIn this notebook, we've demonstrated how to fine-tune OpenAI models for specific use-cases. We've also demonstrated how to use Qdrant and Few-Shot Learning to improve the performance of the model.\n\n### Aggregate Results\n\nSo far, we've looked at the results for each scenario separately, i.e. each scenario summed to 100. Let's look at the results as an aggregate to get a broader sense of how the model is performing:\n\n| Category | Base | Fine-Tuned | Fine-Tuned with Qdrant |\n| --- | --- | --- | --- |\n| Correct | 44% | 32% | 44% |\n| Skipped | 0% | 18% | 5% |\n| Wrong | 9% | 3% | 4% |\n| Hallucination | 47% | 7% | 25% |\n| I don't know | 0% | 40% | 22% |\n\n### Observations\n\n#### Compared to base model\n1. The few shot fine-tuned with Qdrant model is as good as the base model at answering questions where the answer is present in the context. \n2. The few shot fine-tuned with Qdrant model is better at saying \"I don't know\" when the answer is not present in the context.\n3. The few shot fine-tuned with Qdrant model is better at reducing hallucinations."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6a4f252a-a3e7-486c-8bb7-20984dfc6119
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6a4f252a-a3e7-486c-8bb7-20984dfc6119\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"7b96c1f3-71ce-4c5d-b86b-0751a8fc9485\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f79a8e24-2041-48f7-a4da-d8cd41ef2a60\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb\", \"title\": \"Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning\", \"tokens\": 10450, \"retrieve_doc\": false, \"source\": \"openai_cookbooks\"}, \"hash\": \"c055d611d640f319436052228cccdc09f280b9f812ae107ebed5c9d1158545d1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 41903, \"end_char_idx\": 42696, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "ref_doc_id": "7b96c1f3-71ce-4c5d-b86b-0751a8fc9485", "chroma:document": "#### Compared to fine-tuned model\n1. The few shot fine-tuned with Qdrant model gets more correct answers than the fine-tuned model: **83% of the questions are answered correctly vs 60%** for the fine-tuned model\n2. The few shot fine-tuned with Qdrant model is better at deciding when to say \"I don't know\" when the answer is not present in the context. **34% skip rate for the plain fine-tuning mode, vs 9% for the few shot fine-tuned with Qdrant model**\n\n\nNow, you should be able to:\n\n1. Notice the trade-offs between number of correct answers and hallucinations -- and how training dataset choice influences that!\n2. Fine-tune OpenAI models for specific use-cases and use Qdrant to improve the performance of your RAG model\n3. Get started on how to evaluate the performance of your RAG model"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 2bf76086-1eb6-44c7-960e-ab6a6606bba9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"2bf76086-1eb6-44c7-960e-ab6a6606bba9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"94be41bc-365c-4138-8607-134a67703342\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"745a03d972c117d89502888561536d0cd90c1067ef62b129e3757c30418a3f24\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3332, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "# How to use GPT-4 Vision with Function Calling \n\nThe new GPT-4 Turbo model, available as gpt-4-turbo-2024-04-09 as of April 2024, now enables function calling with vision capabilities, better reasoning and a knowledge cutoff date of Dec 2023. Using images with function calling will unlock multimodal use cases and the ability to use reasoning, allowing you to go beyond OCR and image descriptions.\n\nWe will go through two examples to demonstrate the use of function calling with GPT-4 Turbo with Vision:\n\n1. Simulating a customer service assistant for delivery exception support\n2. Analyzing an organizational chart to extract employee information\n\n### Installation and Setup\n\n\n```python\n!pip install pymupdf --quiet\n!pip install openai --quiet\n!pip install matplotlib --quiet\n# instructor makes it easy to work with function calling\n!pip install instructor --quiet\n```\n\n\n```python\nimport base64\nimport os\nfrom enum import Enum\nfrom io import BytesIO\nfrom typing import Iterable\nfrom typing import List\nfrom typing import Literal, Optional\n\nimport fitz\n# Instructor is powered by Pydantic, which is powered by type hints. Schema validation, prompting is controlled by type annotations\nimport instructor\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom IPython.display import display\nfrom PIL import Image\nfrom openai import OpenAI\nfrom pydantic import BaseModel, Field\n```\n\n## 1. Simulating a customer service assistant for delivery exception support\nWe will simulate a customer service assistant for a delivery service that is equipped to analyze images of packages. The assistant will perform the following actions based on the image analysis:\n- If a package appears damaged in the image, automatically process a refund according to policy.\n- If the package looks wet, initiate a replacement.\n- If the package appears normal and not damaged, escalate to an agent.\n\nLet's look at the sample images of packages that the customer service assistant will analyze to determine the appropriate action. We will encode the images as base64 strings for processing by the model.\n\n\n```python\n# Function to encode the image as base64\ndef encode_image(image_path: str):\n    # check if the image exists\n    if not os.path.exists(image_path):\n        raise FileNotFoundError(f\"Image file not found: {image_path}\")\n    with open(image_path, \"rb\") as image_file:\n        return base64.b64encode(image_file.read()).decode('utf-8')\n\n\n# Sample images for testing\nimage_dir = \"images\"\n\n# encode all images within the directory\nimage_files = os.listdir(image_dir)\nimage_data = {}\nfor image_file in image_files:\n    image_path = os.path.join(image_dir, image_file)\n    # encode the image with key as the image file name\n    image_data[image_file.split('.')[0]] = encode_image(image_path)\n    print(f\"Encoded image: {image_file}\")\n\n\ndef display_images(image_data: dict):\n    fig, axs = plt.subplots(1, 3, figsize=(18, 6))\n    for i, (key, value) in enumerate(image_data.items()):\n        img = Image.open(BytesIO(base64.b64decode(value)))\n        ax = axs[i]\n        ax.imshow(img)\n        ax.axis(\"off\")\n        ax.set_title(key)\n    plt.tight_layout()\n    plt.show()\n\n\ndisplay_images(image_data)\n```\n\n    Encoded image: wet_package.jpg\n    Encoded image: damaged_package.jpg\n    Encoded image: normal_package.jpg\n\n\n\n    \n![png](output_6_1.png)"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 94be41bc-365c-4138-8607-134a67703342
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"94be41bc-365c-4138-8607-134a67703342\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2bf76086-1eb6-44c7-960e-ab6a6606bba9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"eb95dec29c0b405d5b4bdbb826ad97a7a7318a3aac39b30ef15d34a7e575a7af\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c5fec1fa-68fc-465c-a633-91631c07857d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d3d45c8c1b85f18165f8e055d7c526a63c9ad5a0c26532dfbd2b013df157db59\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2082, \"end_char_idx\": 3894, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "```python\n# Function to encode the image as base64\ndef encode_image(image_path: str):\n    # check if the image exists\n    if not os.path.exists(image_path):\n        raise FileNotFoundError(f\"Image file not found: {image_path}\")\n    with open(image_path, \"rb\") as image_file:\n        return base64.b64encode(image_file.read()).decode('utf-8')\n\n\n# Sample images for testing\nimage_dir = \"images\"\n\n# encode all images within the directory\nimage_files = os.listdir(image_dir)\nimage_data = {}\nfor image_file in image_files:\n    image_path = os.path.join(image_dir, image_file)\n    # encode the image with key as the image file name\n    image_data[image_file.split('.')[0]] = encode_image(image_path)\n    print(f\"Encoded image: {image_file}\")\n\n\ndef display_images(image_data: dict):\n    fig, axs = plt.subplots(1, 3, figsize=(18, 6))\n    for i, (key, value) in enumerate(image_data.items()):\n        img = Image.open(BytesIO(base64.b64decode(value)))\n        ax = axs[i]\n        ax.imshow(img)\n        ax.axis(\"off\")\n        ax.set_title(key)\n    plt.tight_layout()\n    plt.show()\n\n\ndisplay_images(image_data)\n```\n\n    Encoded image: wet_package.jpg\n    Encoded image: damaged_package.jpg\n    Encoded image: normal_package.jpg\n\n\n\n    \n![png](output_6_1.png)\n    \n\n\nWe have successfully encoded the sample images as base64 strings and displayed them. The customer service assistant will analyze these images to determine the appropriate action based on the package condition.\n\nLet's now define the functions/tools for order processing, such as escalating an order to an agent, refunding an order, and replacing an order. We will create placeholder functions to simulate the processing of these actions based on the identified tools. We will be using Pydantic models to define the structure of the data for order actions."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c5fec1fa-68fc-465c-a633-91631c07857d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c5fec1fa-68fc-465c-a633-91631c07857d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"94be41bc-365c-4138-8607-134a67703342\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"d1c6f555131f1198b7d972d95fa39784bc9dd3ca5bcf54d69425c421acabae96\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f3dcf7e4-16a2-457b-b700-49606fd63c46\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"601b0f363b96725fb8e2aefc4305a30e273ca81d01b026fe6eca2561b435530a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2426, \"end_char_idx\": 6996, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "# Sample images for testing\nimage_dir = \"images\"\n\n# encode all images within the directory\nimage_files = os.listdir(image_dir)\nimage_data = {}\nfor image_file in image_files:\n    image_path = os.path.join(image_dir, image_file)\n    # encode the image with key as the image file name\n    image_data[image_file.split('.')[0]] = encode_image(image_path)\n    print(f\"Encoded image: {image_file}\")\n\n\ndef display_images(image_data: dict):\n    fig, axs = plt.subplots(1, 3, figsize=(18, 6))\n    for i, (key, value) in enumerate(image_data.items()):\n        img = Image.open(BytesIO(base64.b64decode(value)))\n        ax = axs[i]\n        ax.imshow(img)\n        ax.axis(\"off\")\n        ax.set_title(key)\n    plt.tight_layout()\n    plt.show()\n\n\ndisplay_images(image_data)\n```\n\n    Encoded image: wet_package.jpg\n    Encoded image: damaged_package.jpg\n    Encoded image: normal_package.jpg\n\n\n\n    \n![png](output_6_1.png)\n    \n\n\nWe have successfully encoded the sample images as base64 strings and displayed them. The customer service assistant will analyze these images to determine the appropriate action based on the package condition.\n\nLet's now define the functions/tools for order processing, such as escalating an order to an agent, refunding an order, and replacing an order. We will create placeholder functions to simulate the processing of these actions based on the identified tools. We will be using Pydantic models to define the structure of the data for order actions.\n\n\n\n```python\nMODEL = \"gpt-4-turbo-2024-04-09\"\n\nclass Order(BaseModel):\n    \"\"\"Represents an order with details such as order ID, customer name, product name, price, status, and delivery date.\"\"\"\n    order_id: str = Field(..., description=\"The unique identifier of the order\")\n    product_name: str = Field(..., description=\"The name of the product\")\n    price: float = Field(..., description=\"The price of the product\")\n    status: str = Field(..., description=\"The status of the order\")\n    delivery_date: str = Field(..., description=\"The delivery date of the order\")\n# Placeholder functions for order processing\n\ndef get_order_details(order_id):\n    # Placeholder function to retrieve order details based on the order ID\n    return Order(\n        order_id=order_id,\n        product_name=\"Product X\",\n        price=100.0,\n        status=\"Delivered\",\n        delivery_date=\"2024-04-10\",\n    )\n\ndef escalate_to_agent(order: Order, message: str):\n    # Placeholder function to escalate the order to a human agent\n    return f\"Order {order.order_id} has been escalated to an agent with message: `{message}`\"\n\ndef refund_order(order: Order):\n    # Placeholder function to process a refund for the order\n    return f\"Order {order.order_id} has been refunded successfully.\"\n\ndef replace_order(order: Order):\n    # Placeholder function to replace the order with a new one\n    return f\"Order {order.order_id} has been replaced with a new order.\"\n\nclass FunctionCallBase(BaseModel):\n    rationale: Optional[str] = Field(..., description=\"The reason for the action.\")\n    image_description: Optional[str] = Field(\n        ..., description=\"The detailed description of the package image.\"\n    )\n    action: Literal[\"escalate_to_agent\", \"replace_order\", \"refund_order\"]\n    message: Optional[str] = Field(\n        ...,\n        description=\"The message to be escalated to the agent if action is escalate_to_agent\",\n    )\n    # Placeholder functions to process the action based on the order ID\n    def __call__(self, order_id):\n        order: Order = get_order_details(order_id=order_id)\n        if self.action == \"escalate_to_agent\":\n            return escalate_to_agent(order, self.message)\n        if self.action == \"replace_order\":\n            return replace_order(order)\n        if self.action == \"refund_order\":\n            return refund_order(order)\n\nclass EscalateToAgent(FunctionCallBase):\n    \"\"\"Escalate to an agent for further assistance.\"\"\"\n    pass\n\nclass OrderActionBase(FunctionCallBase):\n    pass\n\nclass ReplaceOrder(OrderActionBase):\n    \"\"\"Tool call to replace an order.\"\"\"\n    pass\n\nclass RefundOrder(OrderActionBase):\n    \"\"\"Tool call to refund an order.\"\"\"\n    pass\n```\n\n### Simulating user messages and processing the package images\n\nWe will simulate user messages containing the package images and process the images using the GPT-4 Turbo with Vision model. The model will identify the appropriate tool call based on the image analysis and the predefined actions for damaged, wet, or normal packages. We will then process the identified action based on the order ID and display the results."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: f3dcf7e4-16a2-457b-b700-49606fd63c46
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"f3dcf7e4-16a2-457b-b700-49606fd63c46\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c5fec1fa-68fc-465c-a633-91631c07857d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"1f1f0aaa93b8275f4c4d32ed3d7246ca75a7f7e5335920b15e9926effdc5dddf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ae28ae60-828d-4897-80a2-ae5e7f5cc5c9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1d31a0d166c9e36b48e6c7942431e27b88965cc6df32228d4bcf9656f4bdeca2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6999, \"end_char_idx\": 8706, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "```python\n# extract the tool call from the response\nORDER_ID = \"12345\"  # Placeholder order ID for testing\nINSTRUCTION_PROMPT = \"You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools!\"\n\ndef delivery_exception_support_handler(test_image: str):\n    payload = {\n        \"model\": MODEL,\n        \"response_model\": Iterable[RefundOrder | ReplaceOrder | EscalateToAgent],\n        \"tool_choice\": \"auto\",  # automatically select the tool based on the context\n        \"temperature\": 0.0,  # for less diversity in responses\n        \"seed\": 123,  # Set a seed for reproducibility\n    }\n    payload[\"messages\"] = [\n        {\n            \"role\": \"user\",\n            \"content\": INSTRUCTION_PROMPT,\n        },\n        {\n            \"role\": \"user\",\n            \"content\": [\n                {\n                    \"type\": \"image_url\",\n                    \"image_url\": {\n                        \"url\": f\"data:image/jpeg;base64,{image_data[test_image]}\"\n                    }\n                },\n            ],\n        }\n    ]\n    function_calls = instructor.from_openai(\n        OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS\n    ).chat.completions.create(**payload)\n    for tool in function_calls:\n        print(f\"- Tool call: {tool.action} for provided img: {test_image}\")\n        print(f\"- Parameters: {tool}\")\n        print(f\">> Action result: {tool(ORDER_ID)}\")\n        return tool"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ae28ae60-828d-4897-80a2-ae5e7f5cc5c9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ae28ae60-828d-4897-80a2-ae5e7f5cc5c9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f3dcf7e4-16a2-457b-b700-49606fd63c46\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dc526499f2c39f74dc723a92494bf45ffffed42f225fce30fbb98e15a179df8b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a1f38511-2485-4de7-928d-06a3031c15ef\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0161fce6c3cc5d65361fe0b946bc1e4742fcece4127c254e69f4ff93f4535a45\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6999, \"end_char_idx\": 11389, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "```python\n# extract the tool call from the response\nORDER_ID = \"12345\"  # Placeholder order ID for testing\nINSTRUCTION_PROMPT = \"You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools!\"\n\ndef delivery_exception_support_handler(test_image: str):\n    payload = {\n        \"model\": MODEL,\n        \"response_model\": Iterable[RefundOrder | ReplaceOrder | EscalateToAgent],\n        \"tool_choice\": \"auto\",  # automatically select the tool based on the context\n        \"temperature\": 0.0,  # for less diversity in responses\n        \"seed\": 123,  # Set a seed for reproducibility\n    }\n    payload[\"messages\"] = [\n        {\n            \"role\": \"user\",\n            \"content\": INSTRUCTION_PROMPT,\n        },\n        {\n            \"role\": \"user\",\n            \"content\": [\n                {\n                    \"type\": \"image_url\",\n                    \"image_url\": {\n                        \"url\": f\"data:image/jpeg;base64,{image_data[test_image]}\"\n                    }\n                },\n            ],\n        }\n    ]\n    function_calls = instructor.from_openai(\n        OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS\n    ).chat.completions.create(**payload)\n    for tool in function_calls:\n        print(f\"- Tool call: {tool.action} for provided img: {test_image}\")\n        print(f\"- Parameters: {tool}\")\n        print(f\">> Action result: {tool(ORDER_ID)}\")\n        return tool\n\n\nprint(\"Processing delivery exception support for different package images...\")\n\nprint(\"\\n===================== Simulating user message 1 =====================\")\nassert delivery_exception_support_handler(\"damaged_package\").action == \"refund_order\"\n\nprint(\"\\n===================== Simulating user message 2 =====================\")\nassert delivery_exception_support_handler(\"normal_package\").action == \"escalate_to_agent\"\n\nprint(\"\\n===================== Simulating user message 3 =====================\")\nassert delivery_exception_support_handler(\"wet_package\").action == \"replace_order\"\n```\n\n    Processing delivery exception support for different package images...\n    \n    ===================== Simulating user message 1 =====================\n    - Tool call: refund_order for provided img: damaged_package\n    - Parameters: rationale='The package is visibly damaged with significant tears and crushing, indicating potential harm to the contents.' image_description='The package in the image shows extensive damage, including deep creases and tears in the cardboard. The package is also wrapped with extra tape, suggesting prior attempts to secure it after damage.' action='refund_order' message=None\n    >> Action result: Order 12345 has been refunded successfully.\n    \n    ===================== Simulating user message 2 =====================\n    - Tool call: escalate_to_agent for provided img: normal_package\n    - Parameters: rationale='The package appears normal and not damaged, requiring further assistance for any potential issues not visible in the image.' image_description='A cardboard box on a wooden floor, appearing intact and undamaged, with no visible signs of wear, tear, or wetness.' action='escalate_to_agent' message='Please review this package for any issues not visible in the image. The package appears normal and undamaged.'\n    >> Action result: Order 12345 has been escalated to an agent with message: `Please review this package for any issues not visible in the image. The package appears normal and undamaged.`\n    \n    ===================== Simulating user message 3 =====================\n    - Tool call: replace_order for provided img: wet_package\n    - Parameters: rationale='The package appears wet, which may have compromised the contents, especially since it is labeled as fragile.' image_description=\"The package in the image shows significant wetness on the top surface, indicating potential water damage. The box is labeled 'FRAGILE', which suggests that the contents are delicate and may be more susceptible to damage from moisture.\" action='replace_order' message=None\n    >> Action result: Order 12345 has been replaced with a new order."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a1f38511-2485-4de7-928d-06a3031c15ef
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a1f38511-2485-4de7-928d-06a3031c15ef\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ae28ae60-828d-4897-80a2-ae5e7f5cc5c9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"aa1077ed3a231e949968872ba72a9e6105ca7767e76fb2e4ef8f9383eff34089\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c4724204-c7a4-4262-9c16-23d76cdd3608\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a74d0c370790e2b8f38e6374975e7130d77095e5d57d70256b63e9fd766b3d06\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11392, \"end_char_idx\": 14076, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "## 2.  Analyzing an organizational chart to extract employee information\n\nFor the second example, we will analyze an organizational chart image to extract employee information, such as employee names, roles, managers, and manager roles. We will use GPT-4 Turbo with Vision to process the organizational chart image and extract structured data about the employees in the organization. Indeed, function calling lets us go beyond OCR to actually deduce and translate hierarchical relationships within the chart.\n\nWe will start with a sample organizational chart in PDF format that we want to analyze and convert the first page of the PDF to a JPEG image for analysis.\n\n\n```python\n# Function to convert a single page PDF page to a JPEG image\ndef convert_pdf_page_to_jpg(pdf_path: str, output_path: str, page_number=0):\n    if not os.path.exists(pdf_path):\n        raise FileNotFoundError(f\"PDF file not found: {pdf_path}\")\n    doc = fitz.open(pdf_path)\n    page = doc.load_page(page_number)  # 0 is the first page\n    pix = page.get_pixmap()\n    # Save the pixmap as a JPEG\n    pix.save(output_path)\n\n\ndef display_img_local(image_path: str):\n    img = Image.open(image_path)\n    display(img)\n\n\npdf_path = 'data/org-chart-sample.pdf'\noutput_path = 'org-chart-sample.jpg'\n\nconvert_pdf_page_to_jpg(pdf_path, output_path)\ndisplay_img_local(output_path)\n```\n\n\n    \n![png](output_12_0.png)\n    \n\n\nThe organizational chart image has been successfully extracted from the PDF file and displayed. Let's now define a function to analyze the organizational chart image using the new GPT4 Turbo with Vision. The function will extract information about the employees, their roles, and their managers from the image. We will use function/tool calling to specify the input parameters for the organizational structure, such as the employee name, role, and manager's name and role. We will use Pydantic models to define the structure of the data.\n\n\n\n```python\nbase64_img = encode_image(output_path)\n\nclass RoleEnum(str, Enum):\n    \"\"\"Defines possible roles within an organization.\"\"\"\n    CEO = \"CEO\"\n    CTO = \"CTO\"\n    CFO = \"CFO\"\n    COO = \"COO\"\n    EMPLOYEE = \"Employee\"\n    MANAGER = \"Manager\"\n    INTERN = \"Intern\"\n    OTHER = \"Other\"\n\nclass Employee(BaseModel):\n    \"\"\"Represents an employee, including their name, role, and optional manager information.\"\"\"\n    employee_name: str = Field(..., description=\"The name of the employee\")\n    role: RoleEnum = Field(..., description=\"The role of the employee\")\n    manager_name: Optional[str] = Field(None, description=\"The manager's name, if applicable\")\n    manager_role: Optional[RoleEnum] = Field(None, description=\"The manager's role, if applicable\")"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: c4724204-c7a4-4262-9c16-23d76cdd3608
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"c4724204-c7a4-4262-9c16-23d76cdd3608\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a1f38511-2485-4de7-928d-06a3031c15ef\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ce6c5165ed04b2fc741494e193020bde6222e6e5de6ab9752c59f95d3bbe7ecd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8242b2c9-ab33-46d2-b009-081f84c03c64\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8afa97cc5acd2cc2c8a615f65a45c5d50bdf2d608544d1fb1f7bbef4dd07e52b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 12490, \"end_char_idx\": 15969, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "def display_img_local(image_path: str):\n    img = Image.open(image_path)\n    display(img)\n\n\npdf_path = 'data/org-chart-sample.pdf'\noutput_path = 'org-chart-sample.jpg'\n\nconvert_pdf_page_to_jpg(pdf_path, output_path)\ndisplay_img_local(output_path)\n```\n\n\n    \n![png](output_12_0.png)\n    \n\n\nThe organizational chart image has been successfully extracted from the PDF file and displayed. Let's now define a function to analyze the organizational chart image using the new GPT4 Turbo with Vision. The function will extract information about the employees, their roles, and their managers from the image. We will use function/tool calling to specify the input parameters for the organizational structure, such as the employee name, role, and manager's name and role. We will use Pydantic models to define the structure of the data.\n\n\n\n```python\nbase64_img = encode_image(output_path)\n\nclass RoleEnum(str, Enum):\n    \"\"\"Defines possible roles within an organization.\"\"\"\n    CEO = \"CEO\"\n    CTO = \"CTO\"\n    CFO = \"CFO\"\n    COO = \"COO\"\n    EMPLOYEE = \"Employee\"\n    MANAGER = \"Manager\"\n    INTERN = \"Intern\"\n    OTHER = \"Other\"\n\nclass Employee(BaseModel):\n    \"\"\"Represents an employee, including their name, role, and optional manager information.\"\"\"\n    employee_name: str = Field(..., description=\"The name of the employee\")\n    role: RoleEnum = Field(..., description=\"The role of the employee\")\n    manager_name: Optional[str] = Field(None, description=\"The manager's name, if applicable\")\n    manager_role: Optional[RoleEnum] = Field(None, description=\"The manager's role, if applicable\")\n\n\nclass EmployeeList(BaseModel):\n    \"\"\"A list of employees within the organizational structure.\"\"\"\n    employees: List[Employee] = Field(..., description=\"A list of employees\")\n\ndef parse_orgchart(base64_img: str) -> EmployeeList:\n    response = instructor.from_openai(OpenAI()).chat.completions.create(\n        model='gpt-4-turbo',\n        response_model=EmployeeList,\n        messages=[\n            {\n                \"role\": \"user\",\n                \"content\": 'Analyze the given organizational chart and very carefully extract the information.',\n            },\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": f\"data:image/jpeg;base64,{base64_img}\"\n                        }\n                    },\n                ],\n            }\n        ],\n    )\n    return response\n```\n\nNow, we will define a function to parse the response from GPT-4 Turbo with Vision and extract the employee data. We will tabulate the extracted data for easy visualization. Please note that the accuracy of the extracted data may vary based on the complexity and clarity of the input image.\n\n\n```python\n# call the functions to analyze the organizational chart and parse the response\nresult = parse_orgchart(base64_img)\n\n# tabulate the extracted data\ndf = pd.DataFrame([{\n    'employee_name': employee.employee_name,\n    'role': employee.role.value,\n    'manager_name': employee.manager_name,\n    'manager_role': employee.manager_role.value if employee.manager_role else None\n} for employee in result.employees])\n\ndisplay(df)\n```\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 8242b2c9-ab33-46d2-b009-081f84c03c64
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"8242b2c9-ab33-46d2-b009-081f84c03c64\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c4724204-c7a4-4262-9c16-23d76cdd3608\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"6de0ce211bff2774ec83e80ae37710cae4eb9b30088172611d7ec25bcd604c7a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"33b11382-4d5a-41ab-b7c2-89f400539dbc\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ece6b649218241031b7087b48f3aca6603da0a823f8531daa215ea826646111d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14079, \"end_char_idx\": 16658, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "class EmployeeList(BaseModel):\n    \"\"\"A list of employees within the organizational structure.\"\"\"\n    employees: List[Employee] = Field(..., description=\"A list of employees\")\n\ndef parse_orgchart(base64_img: str) -> EmployeeList:\n    response = instructor.from_openai(OpenAI()).chat.completions.create(\n        model='gpt-4-turbo',\n        response_model=EmployeeList,\n        messages=[\n            {\n                \"role\": \"user\",\n                \"content\": 'Analyze the given organizational chart and very carefully extract the information.',\n            },\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": f\"data:image/jpeg;base64,{base64_img}\"\n                        }\n                    },\n                ],\n            }\n        ],\n    )\n    return response\n```\n\nNow, we will define a function to parse the response from GPT-4 Turbo with Vision and extract the employee data. We will tabulate the extracted data for easy visualization. Please note that the accuracy of the extracted data may vary based on the complexity and clarity of the input image.\n\n\n```python\n# call the functions to analyze the organizational chart and parse the response\nresult = parse_orgchart(base64_img)\n\n# tabulate the extracted data\ndf = pd.DataFrame([{\n    'employee_name': employee.employee_name,\n    'role': employee.role.value,\n    'manager_name': employee.manager_name,\n    'manager_role': employee.manager_role.value if employee.manager_role else None\n} for employee in result.employees])\n\ndisplay(df)\n```\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>employee_name</th>\n      <th>role</th>\n      <th>manager_name</th>\n      <th>manager_role</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n      <td>None</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Kim Chun Hei</td>\n      <td>CFO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Chad Gibbons</td>\n      <td>CTO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Chiaki Sato</td>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 33b11382-4d5a-41ab-b7c2-89f400539dbc
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"33b11382-4d5a-41ab-b7c2-89f400539dbc\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8242b2c9-ab33-46d2-b009-081f84c03c64\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"f51b16322cc6018674fe38a22b7f4c1f76d78510b893d952af302601f57a9aa6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6f1f45fe-9d2a-412d-ab9c-6cd29357032a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1a83601e0883f8d7bd158bd92af26ca072184d6a5c88e490c4591e25bed6e27f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 15914, \"end_char_idx\": 17294, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "}\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>employee_name</th>\n      <th>role</th>\n      <th>manager_name</th>\n      <th>manager_role</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n      <td>None</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Kim Chun Hei</td>\n      <td>CFO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Chad Gibbons</td>\n      <td>CTO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Chiaki Sato</td>\n      <td>COO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n      <td>Kim Chun Hei</td>\n      <td>CFO</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n      <td>Chad Gibbons</td>\n      <td>CTO</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n      <td>Chiaki Sato</td>\n      <td>COO</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Drew Feig</td>\n      <td>Employee</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>8</th>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6f1f45fe-9d2a-412d-ab9c-6cd29357032a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6f1f45fe-9d2a-412d-ab9c-6cd29357032a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"33b11382-4d5a-41ab-b7c2-89f400539dbc\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"65e2eca3add52e7be4d52ace1d434031dead81c0df7e4f9ce99de5b32cbfa736\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4a8c0909-029d-4878-8b5e-fd5afba22701\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"969460755f127eb82048844676c1719f5c0e279955c7e0963406270c7455c8d1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16638, \"end_char_idx\": 17968, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "<td>Chiaki Sato</td>\n      <td>COO</td>\n      <td>Juliana Silva</td>\n      <td>CEO</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n      <td>Kim Chun Hei</td>\n      <td>CFO</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n      <td>Chad Gibbons</td>\n      <td>CTO</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n      <td>Chiaki Sato</td>\n      <td>COO</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>Drew Feig</td>\n      <td>Employee</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>Richard Sanchez</td>\n      <td>Employee</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Sacha Dubois</td>\n      <td>Intern</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Olivia Wilson</td>\n      <td>Employee</td>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>Matt Zhang</td>\n      <td>Intern</td>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>Avery Davis</td>\n      <td>Employee</td>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n    </tr>\n    <tr>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 4a8c0909-029d-4878-8b5e-fd5afba22701
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"4a8c0909-029d-4878-8b5e-fd5afba22701\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"25b20c91-35fc-4666-b23f-0f187e4bb3d5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6f1f45fe-9d2a-412d-ab9c-6cd29357032a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb\", \"title\": \"How to use GPT-4 Vision with Function Calling\", \"tokens\": 4334, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"324b5399da40f34ea71171c394ebde9102acca5cdd7d35786a53385a288b8310\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17263, \"end_char_idx\": 18712, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "ref_doc_id": "25b20c91-35fc-4666-b23f-0f187e4bb3d5", "chroma:document": "</tr>\n    <tr>\n      <th>8</th>\n      <td>Richard Sanchez</td>\n      <td>Employee</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>Sacha Dubois</td>\n      <td>Intern</td>\n      <td>Cahaya Dewi</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>Olivia Wilson</td>\n      <td>Employee</td>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>Matt Zhang</td>\n      <td>Intern</td>\n      <td>Shawn Garcia</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>Avery Davis</td>\n      <td>Employee</td>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>Harper Russo</td>\n      <td>Employee</td>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>Taylor Alonso</td>\n      <td>Intern</td>\n      <td>Aaron Loeb</td>\n      <td>Manager</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\nThe extracted data from the organizational chart has been successfully parsed and displayed in a DataFrame. This approach allows us to leverage GPT-4 Turbo with Vision capabilities to extract structured information from images, such as organizational charts and diagrams, and process the data for further analysis. By using function calling, we can extend the functionality of multimodal models to perform specific tasks or call external functions."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 6943251d-ee90-433e-b0df-f1e5a236687b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb", "title": "Load the embeddings", "tokens": 506, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"6943251d-ee90-433e-b0df-f1e5a236687b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 506, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"0c6c670c-1e11-4e6e-96fd-26c5ab3bc478\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 506, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"9755f69133468856653acb6541fd6cd6611153750fa57cfc2357518380cf3dea\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2071, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "0c6c670c-1e11-4e6e-96fd-26c5ab3bc478", "doc_id": "0c6c670c-1e11-4e6e-96fd-26c5ab3bc478", "ref_doc_id": "0c6c670c-1e11-4e6e-96fd-26c5ab3bc478", "chroma:document": "## Visualizing embeddings in W&B\n\nWe will upload the data to [Weights & Biases](http://wandb.ai) and use an [Embedding Projector](https://docs.wandb.ai/ref/app/features/panels/weave/embedding-projector) to visualize the embeddings using common dimension reduction algorithms like PCA, UMAP, and t-SNE. The dataset is created in the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb).\n\n## What is Weights & Biases?\n\n[Weights & Biases](http://wandb.ai) is a machine learning platform used by OpenAI and other ML teams to build better models faster. They use it to quickly track experiments, evaluate model performance, reproduce models, visualize results, and share findings with colleagues.\n\n### 1. Log the data to W&B\n\nWe create a [W&B Table](https://docs.wandb.ai/guides/data-vis/log-tables) with the original data and the embeddings. Each review is a new row and the 1536 embedding floats are given their own column named `emb_{i}`.\n\n\n```python\nimport pandas as pd\nfrom sklearn.manifold import TSNE\nimport numpy as np\nfrom ast import literal_eval\n\n# Load the embeddings\ndatafile_path = \"data/fine_food_reviews_with_embeddings_1k.csv\"\ndf = pd.read_csv(datafile_path)\n\n# Convert to a list of lists of floats\nmatrix = np.array(df.embedding.apply(literal_eval).to_list())\n```\n\n\n```python\nimport wandb\n\noriginal_cols = df.columns[1:-1].tolist()\nembedding_cols = ['emb_'+str(idx) for idx in range(len(matrix[0]))]\ntable_cols = original_cols + embedding_cols\n\nwith wandb.init(project='openai_embeddings'):\n    table = wandb.Table(columns=table_cols)\n    for i, row in enumerate(df.to_dict(orient=\"records\")):\n        original_data = [row[col_name] for col_name in original_cols]\n        embedding_data = matrix[i].tolist()\n        table.add_data(*(original_data + embedding_data))\n    wandb.log({'openai_embedding_table': table})\n```\n\n### 2. Render as 2D Projection\n\nAfter navigating to the W&B run link, we click the \u2699\ufe0f icon in the top right of the Table and change \"Render As:\" to \"Combined 2D Projection\". \n\nExample: http://wandb.me/openai_embeddings"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ae63c4c9-107d-4aff-a7be-3914924d6dc2
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ae63c4c9-107d-4aff-a7be-3914924d6dc2\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d0b7b93-c6cd-4faa-a630-bd979d0132d8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"176e088b-10d7-45d2-81d6-78a3369ba459\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5c238dd0605c3618695e3c739f33801a19df5a659132e2b7841934c4807a5fdf\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1733, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "ref_doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "chroma:document": "# How to automate tasks with functions (S3 bucket example)\n\nThis code demonstrates how to interact with ChatGPT functions to perform tasks related to Amazon S3 buckets. The notebook covers S3 bucket key functionalities such as running simple listing commands, searching for a specific file in all buckets, uploading a file to a bucket, and downloading a file from a bucket. The OpenAI Chat API understands the user instructions, generates the natural language responses, and extracts appropriate function calls based on the user's input.\n\n**Requirements**:\nTo run the notebook generate AWS access key with S3 bucket writing permission and store them in a local environment file alongside the Openai key. The \"`.env`\" file format:\n```\nAWS_ACCESS_KEY_ID=<your-key>\nAWS_SECRET_ACCESS_KEY=<your-key>\nOPENAI_API_KEY=<your-key>\n```\n\n\n```python\n! pip install openai\n! pip install boto3\n! pip install tenacity\n! pip install python-dotenv\n```\n\n\n```python\nfrom openai import OpenAI\nimport json\nimport boto3\nimport os\nimport datetime\nfrom urllib.request import urlretrieve\n\n# load environment variables\nfrom dotenv import load_dotenv\nload_dotenv() \n```\n\n\n\n\n    True\n\n\n\n## Initials\n\n\n```python\nOpenAI.api_key = os.environ.get(\"OPENAI_API_KEY\")\nGPT_MODEL = \"gpt-3.5-turbo\"\n```\n\n\n```python\n# Optional - if you had issues loading the environment file, you can set the AWS values using the below code\n# os.environ['AWS_ACCESS_KEY_ID'] = ''\n# os.environ['AWS_SECRET_ACCESS_KEY'] = ''\n\n# Create S3 client\ns3_client = boto3.client('s3')\n\n# Create openai client\nclient = OpenAI()\n```\n\n## Utilities\n\nTo connect user questions or commands to the appropriate function, we need to provide ChatGPT with the necessary function details and expected parameters."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 176e088b-10d7-45d2-81d6-78a3369ba459
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"176e088b-10d7-45d2-81d6-78a3369ba459\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d0b7b93-c6cd-4faa-a630-bd979d0132d8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ae63c4c9-107d-4aff-a7be-3914924d6dc2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"93ff23939b8b3257c38a002c1c93b5aac64304109c5698dcdbe90890fc0eaa84\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a1a54710-b5a0-46a7-acf4-eda17ef33b2c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"601087bcb95d65bb471ab51e817d00b8f2b784a8275e195e9023b61bb302d294\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 5303, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "ref_doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "chroma:document": "# How to automate tasks with functions (S3 bucket example)\n\nThis code demonstrates how to interact with ChatGPT functions to perform tasks related to Amazon S3 buckets. The notebook covers S3 bucket key functionalities such as running simple listing commands, searching for a specific file in all buckets, uploading a file to a bucket, and downloading a file from a bucket. The OpenAI Chat API understands the user instructions, generates the natural language responses, and extracts appropriate function calls based on the user's input.\n\n**Requirements**:\nTo run the notebook generate AWS access key with S3 bucket writing permission and store them in a local environment file alongside the Openai key. The \"`.env`\" file format:\n```\nAWS_ACCESS_KEY_ID=<your-key>\nAWS_SECRET_ACCESS_KEY=<your-key>\nOPENAI_API_KEY=<your-key>\n```\n\n\n```python\n! pip install openai\n! pip install boto3\n! pip install tenacity\n! pip install python-dotenv\n```\n\n\n```python\nfrom openai import OpenAI\nimport json\nimport boto3\nimport os\nimport datetime\nfrom urllib.request import urlretrieve\n\n# load environment variables\nfrom dotenv import load_dotenv\nload_dotenv() \n```\n\n\n\n\n    True\n\n\n\n## Initials\n\n\n```python\nOpenAI.api_key = os.environ.get(\"OPENAI_API_KEY\")\nGPT_MODEL = \"gpt-3.5-turbo\"\n```\n\n\n```python\n# Optional - if you had issues loading the environment file, you can set the AWS values using the below code\n# os.environ['AWS_ACCESS_KEY_ID'] = ''\n# os.environ['AWS_SECRET_ACCESS_KEY'] = ''\n\n# Create S3 client\ns3_client = boto3.client('s3')\n\n# Create openai client\nclient = OpenAI()\n```\n\n## Utilities\n\nTo connect user questions or commands to the appropriate function, we need to provide ChatGPT with the necessary function details and expected parameters.\n\n\n```python\n# Functions dict to pass S3 operations details for the GPT model\nfunctions = [\n    {   \n        \"type\": \"function\",\n        \"function\":{\n            \"name\": \"list_buckets\",\n            \"description\": \"List all available S3 buckets\",\n            \"parameters\": {\n                \"type\": \"object\",\n                \"properties\": {}\n            }\n        }\n    },\n    {\n        \"type\": \"function\",\n        \"function\":{\n            \"name\": \"list_objects\",\n            \"description\": \"List the objects or files inside a given S3 bucket\",\n            \"parameters\": {\n                \"type\": \"object\",\n                \"properties\": {\n                    \"bucket\": {\"type\": \"string\", \"description\": \"The name of the S3 bucket\"},\n                    \"prefix\": {\"type\": \"string\", \"description\": \"The folder path in the S3 bucket\"},\n                },\n                \"required\": [\"bucket\"],\n            },\n        }\n    },\n    {   \n        \"type\": \"function\",\n        \"function\":{\n            \"name\": \"download_file\",\n            \"description\": \"Download a specific file from an S3 bucket to a local distribution folder.\",\n            \"parameters\": {\n                \"type\": \"object\",\n                \"properties\": {\n                    \"bucket\": {\"type\": \"string\", \"description\": \"The name of the S3 bucket\"},\n                    \"key\": {\"type\": \"string\", \"description\": \"The path to the file inside the bucket\"},\n                    \"directory\": {\"type\": \"string\", \"description\": \"The local destination directory to download the file, should be specificed by the user.\"},\n                },\n                \"required\": [\"bucket\", \"key\", \"directory\"],\n            }\n        }\n    },\n    {\n        \"type\": \"function\",\n        \"function\":{\n            \"name\": \"upload_file\",\n            \"description\": \"Upload a file to an S3 bucket\",\n            \"parameters\": {\n                \"type\": \"object\",\n                \"properties\": {\n                    \"source\": {\"type\": \"string\", \"description\": \"The local source path or remote URL\"},\n                    \"bucket\": {\"type\": \"string\", \"description\": \"The name of the S3 bucket\"},\n                    \"key\": {\"type\": \"string\", \"description\": \"The path to the file inside the bucket\"},\n                    \"is_remote_url\": {\"type\": \"boolean\", \"description\": \"Is the provided source a URL (True) or local path (False)\"},\n                },\n                \"required\": [\"source\", \"bucket\", \"key\", \"is_remote_url\"],\n            }\n        }\n    },\n    {\n        \"type\": \"function\",\n        \"function\":{\n            \"name\": \"search_s3_objects\",\n            \"description\": \"Search for a specific file name inside an S3 bucket\",\n            \"parameters\": {\n                \"type\": \"object\",\n                \"properties\": {\n                    \"search_name\": {\"type\": \"string\", \"description\": \"The name of the file you want to search for\"},\n                    \"bucket\": {\"type\": \"string\", \"description\": \"The name of the S3 bucket\"},\n                    \"prefix\": {\"type\": \"string\", \"description\": \"The folder path in the S3 bucket\"},\n                    \"exact_match\": {\"type\": \"boolean\", \"description\": \"Set exact_match to True if the search should match the exact file name. Set exact_match to False to compare part of the file name string (the file contains)\"}\n                },\n                \"required\": [\"search_name\"],\n            },\n        }\n    }\n]\n```\n\nCreate helper functions to interact with the S3 service, such as listing buckets, listing objects, downloading and uploading files, and searching for specific files."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a1a54710-b5a0-46a7-acf4-eda17ef33b2c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a1a54710-b5a0-46a7-acf4-eda17ef33b2c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d0b7b93-c6cd-4faa-a630-bd979d0132d8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"176e088b-10d7-45d2-81d6-78a3369ba459\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ae09f84e5de46a76abc534f5b1485f579006d62c1067bd884cff374dd7e64986\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d15c91e5-0fdb-4573-9f0b-a23a721f838e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"dc11f89f7730b179f1533250e1d70834a4a7ce12f54f9762df55c9ff31349b0d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5306, \"end_char_idx\": 8423, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "ref_doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "chroma:document": "```python\ndef datetime_converter(obj):\n    if isinstance(obj, datetime.datetime):\n        return obj.isoformat()\n    raise TypeError(f\"Object of type {obj.__class__.__name__} is not JSON serializable\")\n```\n\n\n```python\ndef list_buckets():\n    response = s3_client.list_buckets()\n    return json.dumps(response['Buckets'], default=datetime_converter)\n\ndef list_objects(bucket, prefix=''):\n    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)\n    return json.dumps(response.get('Contents', []), default=datetime_converter)\n\ndef download_file(bucket, key, directory):\n    \n    filename = os.path.basename(key)\n    \n    # Resolve destination to the correct file path\n    destination = os.path.join(directory, filename)\n    \n    s3_client.download_file(bucket, key, destination)\n    return json.dumps({\"status\": \"success\", \"bucket\": bucket, \"key\": key, \"destination\": destination})\n\ndef upload_file(source, bucket, key, is_remote_url=False):\n    if is_remote_url:\n        file_name = os.path.basename(source)\n        urlretrieve(source, file_name)\n        source = file_name\n       \n    s3_client.upload_file(source, bucket, key)\n    return json.dumps({\"status\": \"success\", \"source\": source, \"bucket\": bucket, \"key\": key})\n\ndef search_s3_objects(search_name, bucket=None, prefix='', exact_match=True):\n    search_name = search_name.lower()\n    \n    if bucket is None:\n        buckets_response = json.loads(list_buckets())\n        buckets = [bucket_info[\"Name\"] for bucket_info in buckets_response]\n    else:\n        buckets = [bucket]\n\n    results = []\n\n    for bucket_name in buckets:\n        objects_response = json.loads(list_objects(bucket_name, prefix))\n        if exact_match:\n            bucket_results = [obj for obj in objects_response if search_name == obj['Key'].lower()]\n        else:\n            bucket_results = [obj for obj in objects_response if search_name in obj['Key'].lower()]\n\n        if bucket_results:\n            results.extend([{\"Bucket\": bucket_name, \"Object\": obj} for obj in bucket_results])\n\n    return json.dumps(results)\n```\n\nThe below dictionary connects the name with the function to use it for execution based on ChatGPT responses.\n\n\n```python\navailable_functions = {\n    \"list_buckets\": list_buckets,\n    \"list_objects\": list_objects,\n    \"download_file\": download_file,\n    \"upload_file\": upload_file,\n    \"search_s3_objects\": search_s3_objects\n}\n```\n\n## ChatGPT\n\n\n```python\ndef chat_completion_request(messages, functions=None, function_call='auto', \n                            model_name=GPT_MODEL):\n    \n    if functions is not None:\n        return client.chat.completions.create(\n            model=model_name,\n            messages=messages,\n            tools=functions,\n            tool_choice=function_call)\n    else:\n        return client.chat.completions.create(\n            model=model_name,\n            messages=messages)\n```\n\n### Conversation flow\n\nCreate a main function for the chatbot, which takes user input, sends it to the OpenAI Chat API, receives a response, executes any function calls generated by the API, and returns a final response to the user."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: d15c91e5-0fdb-4573-9f0b-a23a721f838e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"d15c91e5-0fdb-4573-9f0b-a23a721f838e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d0b7b93-c6cd-4faa-a630-bd979d0132d8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a1a54710-b5a0-46a7-acf4-eda17ef33b2c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"10d0d36e63e0a1bab8138097d8ef723cc7cb0bd1f4eac2d66a7d61a385b1d910\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"229bd745-6aa8-441d-89e6-58781908e628\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a0e08ba5183d28fe846d2380010a8c01ccd9e5d2fa2e3c7295f3f7ff485a7090\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7486, \"end_char_idx\": 10861, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "ref_doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "chroma:document": "```python\navailable_functions = {\n    \"list_buckets\": list_buckets,\n    \"list_objects\": list_objects,\n    \"download_file\": download_file,\n    \"upload_file\": upload_file,\n    \"search_s3_objects\": search_s3_objects\n}\n```\n\n## ChatGPT\n\n\n```python\ndef chat_completion_request(messages, functions=None, function_call='auto', \n                            model_name=GPT_MODEL):\n    \n    if functions is not None:\n        return client.chat.completions.create(\n            model=model_name,\n            messages=messages,\n            tools=functions,\n            tool_choice=function_call)\n    else:\n        return client.chat.completions.create(\n            model=model_name,\n            messages=messages)\n```\n\n### Conversation flow\n\nCreate a main function for the chatbot, which takes user input, sends it to the OpenAI Chat API, receives a response, executes any function calls generated by the API, and returns a final response to the user.\n\n\n```python\ndef run_conversation(user_input, topic=\"S3 bucket functions.\", is_log=False):\n\n    system_message=f\"Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous. If the user ask question not related to {topic} response your scope is {topic} only.\"\n    \n    messages = [{\"role\": \"system\", \"content\": system_message},\n                {\"role\": \"user\", \"content\": user_input}]\n    \n    # Call the model to get a response\n    response = chat_completion_request(messages, functions=functions)\n    response_message = response.choices[0].message\n    \n    if is_log:\n        print(response.choices)\n    \n    # check if GPT wanted to call a function\n    if response_message.tool_calls:\n        function_name = response_message.tool_calls[0].function.name\n        function_args = json.loads(response_message.tool_calls[0].function.arguments)\n        \n        # Call the function\n        function_response = available_functions[function_name](**function_args)\n        \n        # Add the response to the conversation\n        messages.append(response_message)\n        messages.append({\n            \"role\": \"tool\",\n            \"content\": function_response,\n            \"tool_call_id\": response_message.tool_calls[0].id,\n        })\n        \n        # Call the model again to summarize the results\n        second_response = chat_completion_request(messages)\n        final_message = second_response.choices[0].message.content\n    else:\n        final_message = response_message.content\n\n    return final_message\n```\n\n### S3 bucket bot testing\nIn the following examples, make sure to replace the placeholders such as `<file_name>`, `<bucket_name>`, and `<directory_path>` with your specific values before execution.\n\n#### Listing and searching\n\nLet's start by listing all the available buckets.\n\n\n```python\nprint(run_conversation('list my S3 buckets'))\n```\n\nYou can ask the assistant to search for a specific file name either in all the buckets or in a specific one.\n\n\n```python\nsearch_file = '<file_name>'\nprint(run_conversation(f'search for a file {search_file} in all buckets'))\n```\n\n\n```python\nsearch_word = '<file_name_part>'\nbucket_name = '<bucket_name>'\nprint(run_conversation(f'search for a file contains {search_word} in {bucket_name}'))\n```\n\nThe model is expected to clarify the ask from the user in case of ambiguity in the parameters values as described in the system message."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 229bd745-6aa8-441d-89e6-58781908e628
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"229bd745-6aa8-441d-89e6-58781908e628\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3d0b7b93-c6cd-4faa-a630-bd979d0132d8\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d15c91e5-0fdb-4573-9f0b-a23a721f838e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb\", \"title\": \"How to automate tasks with functions (S3 bucket example)\", \"tokens\": 2600, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"3298cbc60ac567678ff57c9d8182e02c7b68caa30ffb7eeb449bf0e257457fbb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10270, \"end_char_idx\": 12140, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "ref_doc_id": "3d0b7b93-c6cd-4faa-a630-bd979d0132d8", "chroma:document": "```python\nprint(run_conversation('list my S3 buckets'))\n```\n\nYou can ask the assistant to search for a specific file name either in all the buckets or in a specific one.\n\n\n```python\nsearch_file = '<file_name>'\nprint(run_conversation(f'search for a file {search_file} in all buckets'))\n```\n\n\n```python\nsearch_word = '<file_name_part>'\nbucket_name = '<bucket_name>'\nprint(run_conversation(f'search for a file contains {search_word} in {bucket_name}'))\n```\n\nThe model is expected to clarify the ask from the user in case of ambiguity in the parameters values as described in the system message.\n\n\n```python\nprint(run_conversation('search for a file'))\n```\n\n    Sure, to help me find what you're looking for, could you please provide the name of the file you want to search for and the name of the S3 bucket? Also, should the search match the file name exactly, or should it also consider partial matches?\n\n\n#### Validate edge cases\n\nWe also instructed the model to reject irrelevant tasks. Let's test it out and see how it works in action.\n\n\n```python\n# the model should not answer details not related to the scope\nprint(run_conversation('what is the weather today'))\n```\n\n    Apologies for the misunderstanding, but I am only able to assist with S3 bucket functions. Can you please ask a question related to S3 bucket functions?\n\n\nThe provided functions are not limited to just retrieving information. They can also assist the user in uploading or downloading files.\n\n#### Download a file\n\n\n```python\nsearch_file = '<file_name>'\nbucket_name = '<bucket_name>'\nlocal_directory = '<directory_path>'\nprint(run_conversation(f'download {search_file} from {bucket_name} bucket to {local_directory} directory'))\n```\n\n#### Upload a file\n\n\n```python\nlocal_file = '<file_name>'\nbucket_name = '<bucket_name>'\nprint(run_conversation(f'upload {local_file} to {bucket_name} bucket'))\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 115930ee-4260-407c-9ec4-8741355430b3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"115930ee-4260-407c-9ec4-8741355430b3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"5ee2f7a0-9254-4f59-9468-fbfa23bedb80\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"99155a013f90dc7ac8d4103ce716eee1247240b046cb70f460ba84886c0e25ff\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2463, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "## Visualizing the embeddings in Kangas\n\nIn this Jupyter Notebook, we construct a Kangas DataGrid containing the data and projections of the embeddings into 2 dimensions.\n\n## What is Kangas?\n\n[Kangas](https://github.com/comet-ml/kangas/) as an open source, mixed-media, dataframe-like tool for data scientists. It was developed by [Comet](https://comet.com/), a company designed to help reduce the friction of moving models into production. \n\n### 1. Setup\n\nTo get started, we pip install kangas, and import it.\n\n\n```python\n%pip install kangas --quiet\n```\n\n\n```python\nimport kangas as kg\n```\n\n### 2. Constructing a Kangas DataGrid\n\nWe create a Kangas Datagrid with the original data and the embeddings. The data is composed of a rows of reviews, and the embeddings are composed of 1536 floating-point values. In this example, we get the data directly from github, in case you aren't running this notebook inside OpenAI's repo.\n\nWe use Kangas to read the CSV file into a DataGrid for further processing.\n\n\n```python\ndata = kg.read_csv(\"https://raw.githubusercontent.com/openai/openai-cookbook/main/examples/data/fine_food_reviews_with_embeddings_1k.csv\")\n```\n\n    Loading CSV file 'fine_food_reviews_with_embeddings_1k.csv'...\n\n\n    1001it [00:00, 2412.90it/s]\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1000/1000 [00:00<00:00, 2899.16it/s]\n\n\nWe can review the fields of the CSV file:\n\n\n```python\ndata.info()\n```\n\n    DataGrid (in memory)\n        Name   : fine_food_reviews_with_embeddings_1k\n        Rows   : 1,000\n        Columns: 9\n    #   Column                Non-Null Count DataGrid Type       \n    --- -------------------- --------------- --------------------\n    1   Column 1                       1,000 INTEGER             \n    2   ProductId                      1,000 TEXT                \n    3   UserId                         1,000 TEXT                \n    4   Score                          1,000 INTEGER             \n    5   Summary                        1,000 TEXT                \n    6   Text                           1,000 TEXT                \n    7   combined                       1,000 TEXT                \n    8   n_tokens                       1,000 INTEGER             \n    9   embedding                      1,000 TEXT                \n\n\nAnd get a glimpse of the first and last rows:\n\n\n```python\ndata\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 5ee2f7a0-9254-4f59-9468-fbfa23bedb80
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"5ee2f7a0-9254-4f59-9468-fbfa23bedb80\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"115930ee-4260-407c-9ec4-8741355430b3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"eb01b02c99c76902db7d736d3b1f0a0d37b111a951907b0fb4411edab558466f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"96953d65-4ab9-4ee4-ad89-75db56cc8898\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6c56da3694b97000668891868af06ad4ef2cd5b6fc8871ebe728ccfe9bf05576\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1004, \"end_char_idx\": 3595, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "```python\ndata = kg.read_csv(\"https://raw.githubusercontent.com/openai/openai-cookbook/main/examples/data/fine_food_reviews_with_embeddings_1k.csv\")\n```\n\n    Loading CSV file 'fine_food_reviews_with_embeddings_1k.csv'...\n\n\n    1001it [00:00, 2412.90it/s]\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1000/1000 [00:00<00:00, 2899.16it/s]\n\n\nWe can review the fields of the CSV file:\n\n\n```python\ndata.info()\n```\n\n    DataGrid (in memory)\n        Name   : fine_food_reviews_with_embeddings_1k\n        Rows   : 1,000\n        Columns: 9\n    #   Column                Non-Null Count DataGrid Type       \n    --- -------------------- --------------- --------------------\n    1   Column 1                       1,000 INTEGER             \n    2   ProductId                      1,000 TEXT                \n    3   UserId                         1,000 TEXT                \n    4   Score                          1,000 INTEGER             \n    5   Summary                        1,000 TEXT                \n    6   Text                           1,000 TEXT                \n    7   combined                       1,000 TEXT                \n    8   n_tokens                       1,000 INTEGER             \n    9   embedding                      1,000 TEXT                \n\n\nAnd get a glimpse of the first and last rows:\n\n\n```python\ndata\n```\n\n\n\n\n<table><th colspan='1' >          row-id </th> <th colspan='1' >        Column 1 </th> <th colspan='1' >       ProductId </th> <th colspan='1' >          UserId </th> <th colspan='1' >           Score </th> <th colspan='1' >         Summary </th> <th colspan='1' >            Text </th> <th colspan='1' >        combined </th> <th colspan='1' >        n_tokens </th> <th colspan='1' >       embedding </th> <tr>\n<td colspan='1' >               1 </td> <td colspan='1' >               0 </td> <td colspan='1' >      B003XPF9BO </td> <td colspan='1' >  A3R7JR3FMEBXQB </td> <td colspan='1' >               5 </td> <td colspan='1' > where does one  </td> <td colspan='1' > Wanted to save  </td> <td colspan='1' > Title: where do </td> <td colspan='1' >              52 </td> <td colspan='1' > [0.007018072064 </td> <tr>\n<td colspan='1' >               2 </td> <td colspan='1' >             297 </td> <td colspan='1' >      B003VXHGPK </td> <td colspan='1' >  A21VWSCGW7UUAR </td> <td colspan='1' >               4 </td> <td colspan='1' > Good, but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good,"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 96953d65-4ab9-4ee4-ad89-75db56cc8898
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"96953d65-4ab9-4ee4-ad89-75db56cc8898\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"5ee2f7a0-9254-4f59-9468-fbfa23bedb80\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"97259f55fe468c8190a4131223366207f2bd4199bf592627cb7d897cfae0f239\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e342b8be-66b8-49c4-b0d2-111fec398626\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"60bdff6a84fa0ce52bc6466d535d76a9881b8113a5d9f83f62075ba131967a28\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2468, \"end_char_idx\": 4477, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "<table><th colspan='1' >          row-id </th> <th colspan='1' >        Column 1 </th> <th colspan='1' >       ProductId </th> <th colspan='1' >          UserId </th> <th colspan='1' >           Score </th> <th colspan='1' >         Summary </th> <th colspan='1' >            Text </th> <th colspan='1' >        combined </th> <th colspan='1' >        n_tokens </th> <th colspan='1' >       embedding </th> <tr>\n<td colspan='1' >               1 </td> <td colspan='1' >               0 </td> <td colspan='1' >      B003XPF9BO </td> <td colspan='1' >  A3R7JR3FMEBXQB </td> <td colspan='1' >               5 </td> <td colspan='1' > where does one  </td> <td colspan='1' > Wanted to save  </td> <td colspan='1' > Title: where do </td> <td colspan='1' >              52 </td> <td colspan='1' > [0.007018072064 </td> <tr>\n<td colspan='1' >               2 </td> <td colspan='1' >             297 </td> <td colspan='1' >      B003VXHGPK </td> <td colspan='1' >  A21VWSCGW7UUAR </td> <td colspan='1' >               4 </td> <td colspan='1' > Good, but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good, bu </td> <td colspan='1' >             178 </td> <td colspan='1' > [-0.00314055196 </td> <tr>\n<td colspan='1' >               3 </td> <td colspan='1' >             296 </td> <td colspan='1' >      B008JKTTUA </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>\n<td colspan='1' >               4 </td> <td colspan='1' >             295 </td> <td colspan='1' >      B000LKTTTW </td> <td colspan='1' >  A14MQ40CCU8B13 </td> <td colspan='1' >               5 </td> <td colspan='1' > Best tomato sou </td> <td colspan='1' > I have a hard t </td> <td colspan='1' > Title: Best tom </td> <td colspan='1' >             111 </td> <td colspan='1' > [-0."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: e342b8be-66b8-49c4-b0d2-111fec398626
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"e342b8be-66b8-49c4-b0d2-111fec398626\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"96953d65-4ab9-4ee4-ad89-75db56cc8898\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"29b10349a3fcb01e5cf774d02bb2604090808094eae77661698caffdd504cd15\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ddb4e91e-31e9-4eae-a60d-a3e46461caa9\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"969c92fd9b6fae0ac217144b2ec99033930de793a03be0b417be601d85f7c386\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good, bu </td> <td colspan='1' >             178 </td> <td colspan='1' > [-0.00314055196 </td> <tr>\n<td colspan='1' >               3 </td> <td colspan='1' >             296 </td> <td colspan='1' >      B008JKTTUA </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>\n<td colspan='1' >               4 </td> <td colspan='1' >             295 </td> <td colspan='1' >      B000LKTTTW </td> <td colspan='1' >  A14MQ40CCU8B13 </td> <td colspan='1' >               5 </td> <td colspan='1' > Best tomato sou </td> <td colspan='1' > I have a hard t </td> <td colspan='1' > Title: Best tom </td> <td colspan='1' >             111 </td> <td colspan='1' > [-0.00139322795 </td> <tr>\n<td colspan='1' >               5 </td> <td colspan='1' >             294 </td> <td colspan='1' >      B001D09KAM </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>\n<tr><td colspan='10' style='text-align: left;'>.</td></tr><td colspan='1' >             996 </td> <td colspan='1' >             623 </td> <td colspan='1' >      B0000CFXYA </td> <td colspan='1' >  A3GS4GWPIBV0NT </td> <td colspan='1' >               1 </td> <td colspan='1' > Strange inflamm </td> <td colspan='1' > Truthfully wasn </td> <td colspan='1' > Title: Strange  </td> <td colspan='1' >             110 </td> <td colspan='1' > [0."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ddb4e91e-31e9-4eae-a60d-a3e46461caa9
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ddb4e91e-31e9-4eae-a60d-a3e46461caa9\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e342b8be-66b8-49c4-b0d2-111fec398626\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"04e066931487bf070cf4257b20244c18df149aae2f1a640bb71f893ca54fd28a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0e2d9df4-eace-43c7-8392-97bc4be1ed44\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"2c3908d5c2f433ac619ad62af16886513f591776e8aa55e2df8f69a05ee059ba\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "00139322795 </td> <tr>\n<td colspan='1' >               5 </td> <td colspan='1' >             294 </td> <td colspan='1' >      B001D09KAM </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>\n<tr><td colspan='10' style='text-align: left;'>.</td></tr><td colspan='1' >             996 </td> <td colspan='1' >             623 </td> <td colspan='1' >      B0000CFXYA </td> <td colspan='1' >  A3GS4GWPIBV0NT </td> <td colspan='1' >               1 </td> <td colspan='1' > Strange inflamm </td> <td colspan='1' > Truthfully wasn </td> <td colspan='1' > Title: Strange  </td> <td colspan='1' >             110 </td> <td colspan='1' > [0.000110913533 </td> <tr>\n<td colspan='1' >             997 </td> <td colspan='1' >             624 </td> <td colspan='1' >      B0001BH5YM </td> <td colspan='1' >   A1BZ3HMAKK0NC </td> <td colspan='1' >               5 </td> <td colspan='1' > My favorite and </td> <td colspan='1' > You've just got </td> <td colspan='1' > Title: My favor </td> <td colspan='1' >              80 </td> <td colspan='1' > [-0.02086931467 </td> <tr>\n<td colspan='1' >             998 </td> <td colspan='1' >             625 </td> <td colspan='1' >      B0009ET7TC </td> <td colspan='1' >  A2FSDQY5AI6TNX </td> <td colspan='1' >               5 </td> <td colspan='1' > My furbabies LO </td> <td colspan='1' > Shake the conta </td> <td colspan='1' > Title: My furba </td> <td colspan='1' >              47 </td> <td colspan='1' > [-0.00974910240 </td> <tr>\n<td colspan='1' >             999 </td> <td colspan='1' >             619 </td> <td colspan='1' >      B007PA32L2 </td> <td colspan='1' >  A15FF2P7RPKH6G </td> <td colspan='1' >               5 </td> <td colspan='1' > got this for th </td> <td colspan='1' > all i have hear </td> <td colspan='1' > Title: got this </td> <td colspan='1' >              50 </td> <td colspan='1' > [-0."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: 0e2d9df4-eace-43c7-8392-97bc4be1ed44
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"0e2d9df4-eace-43c7-8392-97bc4be1ed44\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ddb4e91e-31e9-4eae-a60d-a3e46461caa9\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"e58befb2223c57c1766a4623659097b15b738722d517d8ff7979910ed7b6c37e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a2b13dc6-4267-44b4-a0e7-0aa131711926\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b4315d16f37dcefbee10397aa1bfc1df2ed127ec7827285bbfc01c6dbcb42153\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5752, \"end_char_idx\": 7916, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "02086931467 </td> <tr>\n<td colspan='1' >             998 </td> <td colspan='1' >             625 </td> <td colspan='1' >      B0009ET7TC </td> <td colspan='1' >  A2FSDQY5AI6TNX </td> <td colspan='1' >               5 </td> <td colspan='1' > My furbabies LO </td> <td colspan='1' > Shake the conta </td> <td colspan='1' > Title: My furba </td> <td colspan='1' >              47 </td> <td colspan='1' > [-0.00974910240 </td> <tr>\n<td colspan='1' >             999 </td> <td colspan='1' >             619 </td> <td colspan='1' >      B007PA32L2 </td> <td colspan='1' >  A15FF2P7RPKH6G </td> <td colspan='1' >               5 </td> <td colspan='1' > got this for th </td> <td colspan='1' > all i have hear </td> <td colspan='1' > Title: got this </td> <td colspan='1' >              50 </td> <td colspan='1' > [-0.00521062919 </td> <tr>\n<td colspan='1' >            1000 </td> <td colspan='1' >             999 </td> <td colspan='1' >      B001EQ5GEO </td> <td colspan='1' >  A3VYU0VO6DYV6I </td> <td colspan='1' >               5 </td> <td colspan='1' > I love Maui Cof </td> <td colspan='1' > My first experi </td> <td colspan='1' > Title: I love M </td> <td colspan='1' >             118 </td> <td colspan='1' > [-0.00605782261 </td> <tr>\n<tr>\n<td colspan='10' style=\"text-align: left;\"> [1000 rows x 9 columns] </td> <tr>\n<tr><td colspan='10' style='text-align: left;'></td></tr><tr><td colspan='10' style='text-align: left;'>*  Use DataGrid.save() to save to disk</td></tr><tr><td colspan='10' style='text-align: left;'>** Use DataGrid.show() to start user interface</td></tr></table>\n\n\n\nNow, we create a new DataGrid, converting the numbers into an Embedding:\n\n\n```python\nimport ast # to convert string of a list of numbers into a list of numbers\n\ndg = kg.DataGrid(\n    name=\"openai_embeddings\",\n    columns=data.get_columns(),\n    converters={\"Score\": str},\n)\nfor row in data:\n    embedding = ast.literal_eval(row[8])\n    row[8] = kg.Embedding(\n        embedding, \n        name=str(row[3]), \n        text=\"%s - %.10s\" % (row[3], row[4]),\n        projection=\"umap\",\n    )\n    dg.append(row)\n```\n\nThe new DataGrid now has an Embedding column with proper datatype."}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a2b13dc6-4267-44b4-a0e7-0aa131711926
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a2b13dc6-4267-44b4-a0e7-0aa131711926\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"27b15505-1885-48f9-96f5-a3effb50e0a0\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0e2d9df4-eace-43c7-8392-97bc4be1ed44\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb\", \"title\": \"## Visualizing the embeddings in Kangas\", \"tokens\": 2958, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"eb963b02af751b225cedd8b827386685a73933f34fcafcebec17432d9f5acd3a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6967, \"end_char_idx\": 9839, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "ref_doc_id": "27b15505-1885-48f9-96f5-a3effb50e0a0", "chroma:document": "00605782261 </td> <tr>\n<tr>\n<td colspan='10' style=\"text-align: left;\"> [1000 rows x 9 columns] </td> <tr>\n<tr><td colspan='10' style='text-align: left;'></td></tr><tr><td colspan='10' style='text-align: left;'>*  Use DataGrid.save() to save to disk</td></tr><tr><td colspan='10' style='text-align: left;'>** Use DataGrid.show() to start user interface</td></tr></table>\n\n\n\nNow, we create a new DataGrid, converting the numbers into an Embedding:\n\n\n```python\nimport ast # to convert string of a list of numbers into a list of numbers\n\ndg = kg.DataGrid(\n    name=\"openai_embeddings\",\n    columns=data.get_columns(),\n    converters={\"Score\": str},\n)\nfor row in data:\n    embedding = ast.literal_eval(row[8])\n    row[8] = kg.Embedding(\n        embedding, \n        name=str(row[3]), \n        text=\"%s - %.10s\" % (row[3], row[4]),\n        projection=\"umap\",\n    )\n    dg.append(row)\n```\n\nThe new DataGrid now has an Embedding column with proper datatype.\n\n\n```python\ndg.info()\n```\n\n    DataGrid (in memory)\n        Name   : openai_embeddings\n        Rows   : 1,000\n        Columns: 9\n    #   Column                Non-Null Count DataGrid Type       \n    --- -------------------- --------------- --------------------\n    1   Column 1                       1,000 INTEGER             \n    2   ProductId                      1,000 TEXT                \n    3   UserId                         1,000 TEXT                \n    4   Score                          1,000 TEXT                \n    5   Summary                        1,000 TEXT                \n    6   Text                           1,000 TEXT                \n    7   combined                       1,000 TEXT                \n    8   n_tokens                       1,000 INTEGER             \n    9   embedding                      1,000 EMBEDDING-ASSET     \n\n\nWe simply save the datagrid, and we're done.\n\n\n```python\ndg.save()\n```\n\n### 3. Render 2D Projections\n\nTo render the data directly in the notebook, simply show it. Note that each row contains an embedding projection. \n\nScroll to far right to see embeddings projection per row.\n\nThe color of the point in projection space represents the Score.\n\n\n```python\ndg.show()\n```\n\n\n\n<iframe\n    width=\"100%\"\n    height=\"750px\"\n    src=\"http://127.0.1.1:4000/?datagrid=openai_embeddings.datagrid&timestamp=1685559502.7515423\"\n    frameborder=\"0\"\n    allowfullscreen\n\n></iframe>\n\n\n\nGroup by \"Score\" to see rows of each group.\n\n\n```python\ndg.show(group=\"Score\", sort=\"Score\", rows=5, select=\"Score,embedding\")\n```\n\n\n\n<iframe\n    width=\"100%\"\n    height=\"750px\"\n    src=\"http://127.0.1.1:4000/?datagrid=openai_embeddings.datagrid&timestamp=1685559502.7515423&group=Score&sort=Score&rows=5&select=Score%2Cembedding\"\n    frameborder=\"0\"\n    allowfullscreen\n\n></iframe>\n\n\n\nAn example of this datagrid is hosted here: https://kangas.comet.com/?datagrid=/data/openai_embeddings.datagrid"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: db14ad13-96cf-4251-a931-0d45420d127d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"db14ad13-96cf-4251-a931-0d45420d127d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"054f16dd-1d58-458d-b3b0-1fb93f20d1c5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a5a89cee-03f6-4160-b653-01b6a43beca0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"58041455514a08016ed1c06ea7a8cc66a5f924abf18ed23fec08050c5b4f5ae4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1294, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "ref_doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "chroma:document": "## Visualizing Open AI Embeddings in Atlas\n\nIn this example, we will upload food review embeddings to [Atlas](https://atlas.nomic.ai) to visualize the embeddings.\n\n## What is Atlas?\n\n[Atlas](https://atlas.nomic.ai) is a machine learning tool used to visualize massive datasets of embeddings in your web browser. Upload millions of embeddings to Atlas and interact with them in your web browser or jupyter notebook.\n\n### 1. Login to Atlas.\n\n\n\n```python\n!pip install nomic\n```\n\n    \n\n\n```python\nimport pandas as pd\nimport numpy as np\nfrom ast import literal_eval\n\n# Load the embeddings\ndatafile_path = \"data/fine_food_reviews_with_embeddings_1k.csv\"\ndf = pd.read_csv(datafile_path)\n\n# Convert to a list of lists of floats\nembeddings = np.array(df.embedding.apply(literal_eval).to_list())\ndf = df.drop('embedding', axis=1)\ndf = df.rename(columns={'Unnamed: 0': 'id'})\n\n```\n\n\n```python\nimport nomic\nfrom nomic import atlas\nnomic.login('7xDPkYXSYDc1_ErdTPIcoAR9RNd8YDlkS3nVNXcVoIMZ6') #demo account\n\ndata = df.to_dict('records')\nproject = atlas.map_embeddings(embeddings=embeddings, data=data,\n                               id_field='id',\n                               colorable_fields=['Score'])\nmap = project.maps[0]\n```\n\n    \n\n### 2. Interact with your embeddings in Jupyter\n\n\n```python\nmap\n```"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: a5a89cee-03f6-4160-b653-01b6a43beca0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"a5a89cee-03f6-4160-b653-01b6a43beca0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"054f16dd-1d58-458d-b3b0-1fb93f20d1c5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"db14ad13-96cf-4251-a931-0d45420d127d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"b58f9e9568ae05ced34c2ee368834e0b53b031b922c4c41dcdb7caa75696a214\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ebae1d26-8bc6-4e0d-bb02-36a56d65b9b5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5b0d7e8666a5b807e3bbea3400b6775cad72d42af45bacc0f92b52f9d959d8db\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 2557, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "ref_doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "chroma:document": "## Visualizing Open AI Embeddings in Atlas\n\nIn this example, we will upload food review embeddings to [Atlas](https://atlas.nomic.ai) to visualize the embeddings.\n\n## What is Atlas?\n\n[Atlas](https://atlas.nomic.ai) is a machine learning tool used to visualize massive datasets of embeddings in your web browser. Upload millions of embeddings to Atlas and interact with them in your web browser or jupyter notebook.\n\n### 1. Login to Atlas.\n\n\n\n```python\n!pip install nomic\n```\n\n    \n\n\n```python\nimport pandas as pd\nimport numpy as np\nfrom ast import literal_eval\n\n# Load the embeddings\ndatafile_path = \"data/fine_food_reviews_with_embeddings_1k.csv\"\ndf = pd.read_csv(datafile_path)\n\n# Convert to a list of lists of floats\nembeddings = np.array(df.embedding.apply(literal_eval).to_list())\ndf = df.drop('embedding', axis=1)\ndf = df.rename(columns={'Unnamed: 0': 'id'})\n\n```\n\n\n```python\nimport nomic\nfrom nomic import atlas\nnomic.login('7xDPkYXSYDc1_ErdTPIcoAR9RNd8YDlkS3nVNXcVoIMZ6') #demo account\n\ndata = df.to_dict('records')\nproject = atlas.map_embeddings(embeddings=embeddings, data=data,\n                               id_field='id',\n                               colorable_fields=['Score'])\nmap = project.maps[0]\n```\n\n    \n\n### 2. Interact with your embeddings in Jupyter\n\n\n```python\nmap\n```\n\n\n\n\n\n    <h3>Project: meek-laborer</h3>\n    <script>\n    destroy = function() {\n        document.getElementById(\"iframe463f4614-7689-47e4-b55b-1da0cc679559\").remove()\n    }\n</script>\n\n<h4>Projection ID: 463f4614-7689-47e4-b55b-1da0cc679559</h4>\n<div class=\"actions\">\n    <div id=\"hide\" class=\"action\" onclick=\"destroy()\">Hide embedded project</div>\n    <div class=\"action\" id=\"out\">\n        <a href=\"https://atlas.nomic.ai/map/fddc0e07-97c5-477c-827c-96bca44519aa/463f4614-7689-47e4-b55b-1da0cc679559\" target=\"_blank\">Explore on atlas.nomic.ai</a>\n    </div>\n</div>\n\n<iframe class=\"iframe\" id=\"iframe463f4614-7689-47e4-b55b-1da0cc679559\" allow=\"clipboard-read; clipboard-write\" src=\"https://atlas.nomic.ai/map/fddc0e07-97c5-477c-827c-96bca44519aa/463f4614-7689-47e4-b55b-1da0cc679559\">\n</iframe>\n\n<style>\n    .iframe {\n        /* vh can be **very** large in vscode ipynb. */\n        height: min(75vh, 66vw);\n        width: 100%;\n    }\n</style>\n\n<style>\n    .actions {\n      display: block;\n    }\n    .action {\n      min-height: 18px;\n      margin: 5px;\n      transition: all 500ms ease-in-out;\n    }\n    .action:hover {\n      cursor: pointer;\n    }\n    #hide:hover::after {\n        content: \" X\";\n    }\n    #out:hover::after {\n        content: \"\";\n    }\n</style>"}


created_at: 2024-08-26 20:31:29
topic: persistent://default/default/47f0afae-77ee-4e4a-bcd4-0b050f4de62c
id: ebae1d26-8bc6-4e0d-bb02-36a56d65b9b5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks", "_node_content": "{\"id_\": \"ebae1d26-8bc6-4e0d-bb02-36a56d65b9b5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"054f16dd-1d58-458d-b3b0-1fb93f20d1c5\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a5a89cee-03f6-4160-b653-01b6a43beca0\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb\", \"title\": \"Load the embeddings\", \"tokens\": 776, \"retrieve_doc\": true, \"source\": \"openai_cookbooks\"}, \"hash\": \"729cb66d6a90668ef647874d4b2d45bedb78243aae102e75538732b69c6eaa99\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2564, \"end_char_idx\": 2578, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "ref_doc_id": "054f16dd-1d58-458d-b3b0-1fb93f20d1c5", "chroma:document": "```python\n\n```"}


