# embeddings_queue

created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 0fa8fc55-bdcf-4000-af95-5567bf238769
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb", "title": "Function-calling with an OpenAPI specification", "tokens": 4377, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"0fa8fc55-bdcf-4000-af95-5567bf238769\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b1b5b7-ad6d-45ef-9291-5c677bf8a092\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d9cedaceedb772db16d4de6d0966b9609ad9ea13fe095ad66a98dd8adaf2a5e3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b74e8ffc-3e08-4c4e-8a7b-b2485c568ec4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c92e78ca4f5f886f0d6dc9d7aa26cc2686d4333b279e2177f3f9bb7b23ebbd70\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"a88ec60e-f480-465e-b4e7-88dcb180e47d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8c489a9d682f644d4b336b35d49b13a5b3efd7e843252d84ec0bb95187e468ca\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14246, \"end_char_idx\": 16488, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "ref_doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "chroma:document": "## How to call these functions with GPT\n\n\nNow that we have these function definitions, we can leverage GPT to call them intelligently based on user inputs.\n\nIt's important to note that the chat completions API does not execute the function; instead, it generates the JSON that you can use to call the function in your own code.\n\nFor more information on function-calling, refer to our dedicated [function-calling guide](./How_to_call_functions_with_chat_models.ipynb).\n\n\n\n```python\nSYSTEM_MESSAGE = \"\"\"\nYou are a helpful assistant.\nRespond to the following prompt by using function_call and then summarize actions.\nAsk for clarification if a user request is ambiguous.\n\"\"\"\n\n# Maximum number of function calls allowed to prevent infinite or lengthy loops\nMAX_CALLS = 5\n\n\ndef get_openai_response(functions, messages):\n    return client.chat.completions.create(\n        model=\"gpt-3.5-turbo-16k\",\n        tools=functions,\n        tool_choice=\"auto\",  # \"auto\" means the model can pick between generating a message or calling a function.\n        temperature=0,\n        messages=messages,\n    )\n\n\ndef process_user_instruction(functions, instruction):\n    num_calls = 0\n    messages = [\n        {\"content\": SYSTEM_MESSAGE, \"role\": \"system\"},\n        {\"content\": instruction, \"role\": \"user\"},\n    ]\n\n    while num_calls < MAX_CALLS:\n        response = get_openai_response(functions, messages)\n        message = response.choices[0].message\n        print(message)\n        try:\n            print(f\"\\n>> Function call #: {num_calls + 1}\\n\")\n            pp(message.tool_calls)\n            messages.append(message)\n\n            # For the sake of this example, we'll simply add a message to simulate success.\n            # Normally, you'd want to call the function here, and append the results to messages.\n            messages.append(\n                {\n                    \"role\": \"tool\",\n                    \"content\": \"success\",\n                    \"tool_call_id\": message.tool_calls[0].id,\n                }\n            )\n\n            num_calls += 1\n        except:\n            print(\"\\n>> Message:\\n\")\n            print(message.content)\n            break\n\n    if num_calls >= MAX_CALLS:\n        print(f\"Reached max chained function calls: {MAX_CALLS}\")"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: a88ec60e-f480-465e-b4e7-88dcb180e47d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb", "title": "Function-calling with an OpenAPI specification", "tokens": 4377, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"a88ec60e-f480-465e-b4e7-88dcb180e47d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b1b5b7-ad6d-45ef-9291-5c677bf8a092\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d9cedaceedb772db16d4de6d0966b9609ad9ea13fe095ad66a98dd8adaf2a5e3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0fa8fc55-bdcf-4000-af95-5567bf238769\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"6a4e7baeb0b753d7731dbe063476ab3fb75ade0eb41c4de611e76f0f7807c69c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"948b739b-0887-44cc-b28f-a60e883975f1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"89289eab3468bfa2033e07e37caeab357cb7b632ce0dfe1a781d2fe6dbea0296\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14717, \"end_char_idx\": 18865, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "ref_doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "chroma:document": "```python\nSYSTEM_MESSAGE = \"\"\"\nYou are a helpful assistant.\nRespond to the following prompt by using function_call and then summarize actions.\nAsk for clarification if a user request is ambiguous.\n\"\"\"\n\n# Maximum number of function calls allowed to prevent infinite or lengthy loops\nMAX_CALLS = 5\n\n\ndef get_openai_response(functions, messages):\n    return client.chat.completions.create(\n        model=\"gpt-3.5-turbo-16k\",\n        tools=functions,\n        tool_choice=\"auto\",  # \"auto\" means the model can pick between generating a message or calling a function.\n        temperature=0,\n        messages=messages,\n    )\n\n\ndef process_user_instruction(functions, instruction):\n    num_calls = 0\n    messages = [\n        {\"content\": SYSTEM_MESSAGE, \"role\": \"system\"},\n        {\"content\": instruction, \"role\": \"user\"},\n    ]\n\n    while num_calls < MAX_CALLS:\n        response = get_openai_response(functions, messages)\n        message = response.choices[0].message\n        print(message)\n        try:\n            print(f\"\\n>> Function call #: {num_calls + 1}\\n\")\n            pp(message.tool_calls)\n            messages.append(message)\n\n            # For the sake of this example, we'll simply add a message to simulate success.\n            # Normally, you'd want to call the function here, and append the results to messages.\n            messages.append(\n                {\n                    \"role\": \"tool\",\n                    \"content\": \"success\",\n                    \"tool_call_id\": message.tool_calls[0].id,\n                }\n            )\n\n            num_calls += 1\n        except:\n            print(\"\\n>> Message:\\n\")\n            print(message.content)\n            break\n\n    if num_calls >= MAX_CALLS:\n        print(f\"Reached max chained function calls: {MAX_CALLS}\")\n\n\nUSER_INSTRUCTION = \"\"\"\nInstruction: Get all the events.\nThen create a new event named AGI Party.\nThen delete event with id 2456.\n\"\"\"\n\nprocess_user_instruction(functions, USER_INSTRUCTION)\n\n```\n\n    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_jmlvEyMRMvOtB80adX9RbqIV', function=Function(arguments='{}', name='listEvents'), type='function')])\n    \n    >> Function call #: 1\n    \n    [ChatCompletionMessageToolCall(id='call_jmlvEyMRMvOtB80adX9RbqIV', function=Function(arguments='{}', name='listEvents'), type='function')]\n    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_OOPOY7IHMq3T7Ib71JozlUQJ', function=Function(arguments='{\\n  \"requestBody\": {\\n    \"id\": \"1234\",\\n    \"name\": \"AGI Party\",\\n    \"date\": \"2022-12-31\",\\n    \"location\": \"New York\"\\n  }\\n}', name='createEvent'), type='function')])\n    \n    >> Function call #: 2\n    \n    [ChatCompletionMessageToolCall(id='call_OOPOY7IHMq3T7Ib71JozlUQJ', function=Function(arguments='{\\n  \"requestBody\": {\\n    \"id\": \"1234\",\\n    \"name\": \"AGI Party\",\\n    \"date\": \"2022-12-31\",\\n    \"location\": \"New York\"\\n  }\\n}', name='createEvent'), type='function')]\n    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_Kxluu3fJSOsZNNCn3JIlWAAM', function=Function(arguments='{\\n  \"parameters\": {\\n    \"id\": \"2456\"\\n  }\\n}', name='deleteEvent'), type='function')])\n    \n    >> Function call #: 3\n    \n    [ChatCompletionMessageToolCall(id='call_Kxluu3fJSOsZNNCn3JIlWAAM', function=Function(arguments='{\\n  \"parameters\": {\\n    \"id\": \"2456\"\\n  }\\n}', name='deleteEvent'), type='function')]\n    ChatCompletionMessage(content='Here are the actions I performed:\\n\\n1. Retrieved all the events.\\n2. Created a new event named \"AGI Party\" with the ID \"1234\", scheduled for December 31, 2022, in New York.\\n3. Deleted the event with the ID \"2456\".', role='assistant', function_call=None, tool_calls=None)\n    \n    >> Function call #: 4\n    \n    None\n    \n    >> Message:\n    \n    Here are the actions I performed:\n    \n    1. Retrieved all the events.\n    2. Created a new event named \"AGI Party\" with the ID \"1234\", scheduled for December 31, 2022, in New York.\n    3. Deleted the event with the ID \"2456\"."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 948b739b-0887-44cc-b28f-a60e883975f1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb", "title": "Function-calling with an OpenAPI specification", "tokens": 4377, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"948b739b-0887-44cc-b28f-a60e883975f1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"17b1b5b7-ad6d-45ef-9291-5c677bf8a092\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d9cedaceedb772db16d4de6d0966b9609ad9ea13fe095ad66a98dd8adaf2a5e3\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a88ec60e-f480-465e-b4e7-88dcb180e47d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Function_calling_with_an_OpenAPI_spec.ipynb\", \"title\": \"Function-calling with an OpenAPI specification\", \"tokens\": 4377, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"53bd9ab31d953adb7606a39f2604ffc7c77ec966c9d58533a16274b6842f6c55\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18868, \"end_char_idx\": 19410, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "ref_doc_id": "17b1b5b7-ad6d-45ef-9291-5c677bf8a092", "chroma:document": "### Conclusion\n\nWe have demonstrated how to convert OpenAPI specs into function specifications that can be given to GPT for it to intelligently call them, and shown how these can be chained together to perform complex operations.\n\nPossible extensions of this system could include handling more complex user instructions that require conditional logic or looping, integrating with real APIs to perform actual operations, and improving error handling and validation to ensure the instructions are feasible and the function calls are successful."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 4a6d9cf0-595f-4c21-9efd-e08f4cdaf4af
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"4a6d9cf0-595f-4c21-9efd-e08f4cdaf4af\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3bc75bb7-dd7d-4788-805a-7b39b6c5eab1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f19327d32a838ac329cc673258b0bd4b9f61853e1cb407f151dc3a9154e14c9f\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 1839, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "# How to stream completions\n\nBy default, when you request a completion from the OpenAI, the entire completion is generated before being sent back in a single response.\n\nIf you're generating long completions, waiting for the response can take many seconds.\n\nTo get responses sooner, you can 'stream' the completion as it's being generated. This allows you to start printing or processing the beginning of the completion before the full completion is finished.\n\nTo stream completions, set `stream=True` when calling the chat completions or completions endpoints. This will return an object that streams back the response as [data-only server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format). Extract chunks from the `delta` field rather than the `message` field.\n\n## Downsides\n\nNote that using `stream=True` in a production application makes it more difficult to moderate the content of the completions, as partial completions may be more difficult to evaluate. This may have implications for [approved usage](https://beta.openai.com/docs/usage-guidelines).\n\n## Example code\n\nBelow, this notebook shows:\n1. What a typical chat completion response looks like\n2. What a streaming chat completion response looks like\n3. How much time is saved by streaming a chat completion\n4. How to get token usage data for streamed chat completion response\n\n\n```python\n# !pip install openai\n```\n\n\n```python\n# imports\nimport time  # for measuring time duration of API calls\nfrom openai import OpenAI\nimport os\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```\n\n### 1. What a typical chat completion response looks like\n\nWith a typical ChatCompletions API call, the response is first computed and then returned all at once."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 3bc75bb7-dd7d-4788-805a-7b39b6c5eab1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"3bc75bb7-dd7d-4788-805a-7b39b6c5eab1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4a6d9cf0-595f-4c21-9efd-e08f4cdaf4af\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"8dd351c7a97c390f7420f561ca36dfaaa4fd1f5c8875d2e5a3ca26ae4660aa72\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"812ff326-fc6e-41e0-ab03-f6dd08770fae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"005efff13db7f0180872f8e879cc3d54467c9984564ac5ee8e217521a0e3d4ad\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3477, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "# How to stream completions\n\nBy default, when you request a completion from the OpenAI, the entire completion is generated before being sent back in a single response.\n\nIf you're generating long completions, waiting for the response can take many seconds.\n\nTo get responses sooner, you can 'stream' the completion as it's being generated. This allows you to start printing or processing the beginning of the completion before the full completion is finished.\n\nTo stream completions, set `stream=True` when calling the chat completions or completions endpoints. This will return an object that streams back the response as [data-only server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format). Extract chunks from the `delta` field rather than the `message` field.\n\n## Downsides\n\nNote that using `stream=True` in a production application makes it more difficult to moderate the content of the completions, as partial completions may be more difficult to evaluate. This may have implications for [approved usage](https://beta.openai.com/docs/usage-guidelines).\n\n## Example code\n\nBelow, this notebook shows:\n1. What a typical chat completion response looks like\n2. What a streaming chat completion response looks like\n3. How much time is saved by streaming a chat completion\n4. How to get token usage data for streamed chat completion response\n\n\n```python\n# !pip install openai\n```\n\n\n```python\n# imports\nimport time  # for measuring time duration of API calls\nfrom openai import OpenAI\nimport os\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```\n\n### 1. What a typical chat completion response looks like\n\nWith a typical ChatCompletions API call, the response is first computed and then returned all at once.\n\n\n```python\n# Example of an OpenAI ChatCompletion request\n# https://platform.openai.com/docs/guides/text-generation/chat-completions-api\n\n# record the time before the request is sent\nstart_time = time.time()\n\n# send a ChatCompletion request to count to 100\nresponse = client.chat.completions.create(\n    model='gpt-4o-mini',\n    messages=[\n        {'role': 'user', 'content': 'Count to 100, with a comma between each number and no newlines. E.g., 1, 2, 3, ...'}\n    ],\n    temperature=0,\n)\n# calculate the time it took to receive the response\nresponse_time = time.time() - start_time\n\n# print the time delay and text received\nprint(f\"Full response received {response_time:.2f} seconds after request\")\nprint(f\"Full response received:\\n{response}\")\n\n```\n\n    Full response received 1.88 seconds after request\n    Full response received:\n    ChatCompletion(id='chatcmpl-9lMgdoiMfxVHPDNVCtvXuTWcQ2GGb', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100', role='assistant', function_call=None, tool_calls=None))], created=1721075651, model='gpt-july-test', object='chat.completion', system_fingerprint='fp_e9b8ed65d2', usage=CompletionUsage(completion_tokens=298, prompt_tokens=36, total_tokens=334))"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 812ff326-fc6e-41e0-ab03-f6dd08770fae
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"812ff326-fc6e-41e0-ab03-f6dd08770fae\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3bc75bb7-dd7d-4788-805a-7b39b6c5eab1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"79b3f2b880a759c14e0a6f4f61cf1f57cbd1385ac616ed79073ddfe7c9cef1f2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6aedbbce-1ab9-4c3d-908b-0164331dd917\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ac916f2d332902c67eb053fd839078736f445348dd1e5a82cb4aaad43eed1e4e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3480, \"end_char_idx\": 4744, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "The reply can be extracted with `response.choices[0].message`.\n\nThe content of the reply can be extracted with `response.choices[0].message.content`.\n\n\n```python\nreply = response.choices[0].message\nprint(f\"Extracted reply: \\n{reply}\")\n\nreply_content = response.choices[0].message.content\nprint(f\"Extracted content: \\n{reply_content}\")\n\n```\n\n    Extracted reply: \n    ChatCompletionMessage(content='1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100', role='assistant', function_call=None, tool_calls=None)\n    Extracted content: \n    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 6aedbbce-1ab9-4c3d-908b-0164331dd917
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"6aedbbce-1ab9-4c3d-908b-0164331dd917\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"812ff326-fc6e-41e0-ab03-f6dd08770fae\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"ff82b6f6e5404d684130d000bd1b2122c7e9f852ae7ad48e6019c4a80da0c7c6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cf5726ee-12f4-4264-a83b-9350bddcc37a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ef6f9c1af415f37e91b90b1e154b5ada9453c801582d4aa752af14f769138842\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 4747, \"end_char_idx\": 7494, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "### 2. How to stream a chat completion\n\nWith a streaming API call, the response is sent back incrementally in chunks via an [event stream](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format). In Python, you can iterate over these events with a `for` loop.\n\nLet's see what it looks like:\n\n\n```python\n# Example of an OpenAI ChatCompletion request with stream=True\n# https://platform.openai.com/docs/api-reference/streaming#chat/create-stream\n\n# a ChatCompletion request\nresponse = client.chat.completions.create(\n    model='gpt-4o-mini',\n    messages=[\n        {'role': 'user', 'content': \"What's 1+1? Answer in one word.\"}\n    ],\n    temperature=0,\n    stream=True  # this time, we set stream=True\n)\n\nfor chunk in response:\n    print(chunk)\n    print(chunk.choices[0].delta.content)\n    print(\"****************\")\n```\n\n    ChatCompletionChunk(id='chatcmpl-9lMgfRSWPHcw51s6wxKT1YEO2CKpd', choices=[Choice(delta=ChoiceDelta(content='', function_call=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)], created=1721075653, model='gpt-july-test', object='chat.completion.chunk', system_fingerprint='fp_e9b8ed65d2', usage=None)\n    \n    ****************\n    ChatCompletionChunk(id='chatcmpl-9lMgfRSWPHcw51s6wxKT1YEO2CKpd', choices=[Choice(delta=ChoiceDelta(content='Two', function_call=None, role=None, tool_calls=None), finish_reason=None, index=0, logprobs=None)], created=1721075653, model='gpt-july-test', object='chat.completion.chunk', system_fingerprint='fp_e9b8ed65d2', usage=None)\n    Two\n    ****************\n    ChatCompletionChunk(id='chatcmpl-9lMgfRSWPHcw51s6wxKT1YEO2CKpd', choices=[Choice(delta=ChoiceDelta(content='.', function_call=None, role=None, tool_calls=None), finish_reason=None, index=0, logprobs=None)], created=1721075653, model='gpt-july-test', object='chat.completion.chunk', system_fingerprint='fp_e9b8ed65d2', usage=None)\n    .\n    ****************\n    ChatCompletionChunk(id='chatcmpl-9lMgfRSWPHcw51s6wxKT1YEO2CKpd', choices=[Choice(delta=ChoiceDelta(content=None, function_call=None, role=None, tool_calls=None), finish_reason='stop', index=0, logprobs=None)], created=1721075653, model='gpt-july-test', object='chat.completion.chunk', system_fingerprint='fp_e9b8ed65d2', usage=None)\n    None\n    ****************\n\n\nAs you can see above, streaming responses have a `delta` field rather than a `message` field. `delta` can hold things like:\n- a role token (e.g., `{\"role\": \"assistant\"}`)\n- a content token (e.g., `{\"content\": \"\\n\\n\"}`)\n- nothing (e.g., `{}`), when the stream is over\n\n### 3. How much time is saved by streaming a chat completion\n\nNow let's ask `gpt-4o-mini` to count to 100 again, and see how long it takes."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: cf5726ee-12f4-4264-a83b-9350bddcc37a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"cf5726ee-12f4-4264-a83b-9350bddcc37a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6aedbbce-1ab9-4c3d-908b-0164331dd917\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"a1546d955dce714e8e39259ca3ae15cdb83589e431909b7800e5036186721430\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e9624315-27ee-43b3-b433-08303e309adb\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"779bc69aeff0fdd1ed72b851d81a2391ccd79794595fcc30778b32c2a8521097\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "As you can see above, streaming responses have a `delta` field rather than a `message` field. `delta` can hold things like:\n- a role token (e.g., `{\"role\": \"assistant\"}`)\n- a content token (e.g., `{\"content\": \"\\n\\n\"}`)\n- nothing (e.g., `{}`), when the stream is over\n\n### 3. How much time is saved by streaming a chat completion\n\nNow let's ask `gpt-4o-mini` to count to 100 again, and see how long it takes.\n\n\n```python\n# Example of an OpenAI ChatCompletion request with stream=True\n# https://platform.openai.com/docs/api-reference/streaming#chat/create-stream\n\n# record the time before the request is sent\nstart_time = time.time()\n\n# send a ChatCompletion request to count to 100\nresponse = client.chat.completions.create(\n    model='gpt-4o-mini',\n    messages=[\n        {'role': 'user', 'content': 'Count to 100, with a comma between each number and no newlines. E.g. 1, 2, 3, .'}\n    ],\n    temperature=0,\n    stream=True  # again, we set stream=True\n)\n# create variables to collect the stream of chunks\ncollected_chunks = []\ncollected_messages = []\n# iterate through the stream of events\nfor chunk in response:\n    chunk_time = time.time() - start_time  # calculate the time delay of the chunk\n    collected_chunks.append(chunk)  # save the event response\n    chunk_message = chunk.choices[0].delta.content  # extract the message\n    collected_messages.append(chunk_message)  # save the message\n    print(f\"Message received {chunk_time:.2f} seconds after request: {chunk_message}\")  # print the delay and text\n\n# print the time delay and text received\nprint(f\"Full response received {chunk_time:.2f} seconds after request\")\n# clean None in collected_messages\ncollected_messages = [m for m in collected_messages if m is not None]\nfull_reply_content = ''.join(collected_messages)\nprint(f\"Full conversation received: {full_reply_content}\")\n\n```\n\n    Message received 1.14 seconds after request: \n    Message received 1.14 seconds after request: 1\n    Message received 1.14 seconds after request: ,\n    Message received 1.14 seconds after request:  \n    Message received 1.14 seconds after request: 2\n    Message received 1.16 seconds after request: ,\n    Message received 1.16 seconds after request:  \n    Message received 1.16 seconds after request: 3\n    Message received 1.35 seconds after request: ,\n    Message received 1.35 seconds after request:  \n    Message received 1.35 seconds after request: 4\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 5\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 6\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 7\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e9624315-27ee-43b3-b433-08303e309adb
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e9624315-27ee-43b3-b433-08303e309adb\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cf5726ee-12f4-4264-a83b-9350bddcc37a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"5e755082de09a2f71dd7c1d3a55f81b91506f0bed130f09e58eb354270122fcd\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3a08bcf9-1463-40c4-a5dd-c3bac12753c6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e933aaa3447735fdea2b308e50d5c1fe195176a3537232f889e8a4a1e2295d82\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8531, \"end_char_idx\": 11619, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "2f} seconds after request: {chunk_message}\")  # print the delay and text\n\n# print the time delay and text received\nprint(f\"Full response received {chunk_time:.2f} seconds after request\")\n# clean None in collected_messages\ncollected_messages = [m for m in collected_messages if m is not None]\nfull_reply_content = ''.join(collected_messages)\nprint(f\"Full conversation received: {full_reply_content}\")\n\n```\n\n    Message received 1.14 seconds after request: \n    Message received 1.14 seconds after request: 1\n    Message received 1.14 seconds after request: ,\n    Message received 1.14 seconds after request:  \n    Message received 1.14 seconds after request: 2\n    Message received 1.16 seconds after request: ,\n    Message received 1.16 seconds after request:  \n    Message received 1.16 seconds after request: 3\n    Message received 1.35 seconds after request: ,\n    Message received 1.35 seconds after request:  \n    Message received 1.35 seconds after request: 4\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 5\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 6\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 7\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 8\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 9\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 10\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 11\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 12\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.45 seconds after request: 13\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.45 seconds after request: 14\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.45 seconds after request: 15\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.46 seconds after request: 16\n    Message received 1.46 seconds after request: ,\n    Message received 1.46 seconds after request:  \n    Message received 1.47 seconds after request: 17\n    Message received 1.47 seconds after request: ,\n    Message received 1.47 seconds after request:  \n    Message received 1."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 3a08bcf9-1463-40c4-a5dd-c3bac12753c6
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"3a08bcf9-1463-40c4-a5dd-c3bac12753c6\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e9624315-27ee-43b3-b433-08303e309adb\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"749878ff42746ec3990c13ab9c70d650317c812a0344828dd290c90fddbbbffe\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b351b993-9d63-4108-bd27-509f5c826a44\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f0fc35973dc7c9ea1a318144fb7fa672bb9a80d659ab774b140d9cabad8de82c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10081, \"end_char_idx\": 13108, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "36 seconds after request: 8\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 9\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 10\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 11\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.36 seconds after request: 12\n    Message received 1.36 seconds after request: ,\n    Message received 1.36 seconds after request:  \n    Message received 1.45 seconds after request: 13\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.45 seconds after request: 14\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.45 seconds after request: 15\n    Message received 1.45 seconds after request: ,\n    Message received 1.45 seconds after request:  \n    Message received 1.46 seconds after request: 16\n    Message received 1.46 seconds after request: ,\n    Message received 1.46 seconds after request:  \n    Message received 1.47 seconds after request: 17\n    Message received 1.47 seconds after request: ,\n    Message received 1.47 seconds after request:  \n    Message received 1.49 seconds after request: 18\n    Message received 1.49 seconds after request: ,\n    Message received 1.49 seconds after request:  \n    Message received 1.52 seconds after request: 19\n    Message received 1.52 seconds after request: ,\n    Message received 1.52 seconds after request:  \n    Message received 1.53 seconds after request: 20\n    Message received 1.53 seconds after request: ,\n    Message received 1.53 seconds after request:  \n    Message received 1.55 seconds after request: 21\n    Message received 1.55 seconds after request: ,\n    Message received 1.55 seconds after request:  \n    Message received 1.56 seconds after request: 22\n    Message received 1.56 seconds after request: ,\n    Message received 1.56 seconds after request:  \n    Message received 1.58 seconds after request: 23\n    Message received 1.58 seconds after request: ,\n    Message received 1.58 seconds after request:  \n    Message received 1.59 seconds after request: 24\n    Message received 1.59 seconds after request: ,\n    Message received 1.59 seconds after request:  \n    Message received 1.62 seconds after request: 25\n    Message received 1.62 seconds after request: ,\n    Message received 1.62 seconds after request:  \n    Message received 1.62 seconds after request: 26\n    Message received 1.62 seconds after request: ,\n    Message received 1.62 seconds after request:  \n    Message received 1.65 seconds after request: 27\n    Message received 1.65 seconds after request: ,\n    Message received 1."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b351b993-9d63-4108-bd27-509f5c826a44
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b351b993-9d63-4108-bd27-509f5c826a44\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3a08bcf9-1463-40c4-a5dd-c3bac12753c6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"bd418ed7d1f91a6337cc005f63443e7567be11c13b819fa08723832fb361383a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0c9c193a-e44e-4b63-8762-b01716209cb4\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7af32aeb1fa9366341e3d56383a84ce543ab2fa36fc9625e753f2aceb2538d5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 11549, \"end_char_idx\": 14597, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "Message received 1.47 seconds after request:  \n    Message received 1.49 seconds after request: 18\n    Message received 1.49 seconds after request: ,\n    Message received 1.49 seconds after request:  \n    Message received 1.52 seconds after request: 19\n    Message received 1.52 seconds after request: ,\n    Message received 1.52 seconds after request:  \n    Message received 1.53 seconds after request: 20\n    Message received 1.53 seconds after request: ,\n    Message received 1.53 seconds after request:  \n    Message received 1.55 seconds after request: 21\n    Message received 1.55 seconds after request: ,\n    Message received 1.55 seconds after request:  \n    Message received 1.56 seconds after request: 22\n    Message received 1.56 seconds after request: ,\n    Message received 1.56 seconds after request:  \n    Message received 1.58 seconds after request: 23\n    Message received 1.58 seconds after request: ,\n    Message received 1.58 seconds after request:  \n    Message received 1.59 seconds after request: 24\n    Message received 1.59 seconds after request: ,\n    Message received 1.59 seconds after request:  \n    Message received 1.62 seconds after request: 25\n    Message received 1.62 seconds after request: ,\n    Message received 1.62 seconds after request:  \n    Message received 1.62 seconds after request: 26\n    Message received 1.62 seconds after request: ,\n    Message received 1.62 seconds after request:  \n    Message received 1.65 seconds after request: 27\n    Message received 1.65 seconds after request: ,\n    Message received 1.65 seconds after request:  \n    Message received 1.67 seconds after request: 28\n    Message received 1.67 seconds after request: ,\n    Message received 1.67 seconds after request:  \n    Message received 1.69 seconds after request: 29\n    Message received 1.69 seconds after request: ,\n    Message received 1.69 seconds after request:  \n    Message received 1.80 seconds after request: 30\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 31\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 32\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 33\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 34\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 35\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 36\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.82 seconds after request: 37\n    Message received 1."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 0c9c193a-e44e-4b63-8762-b01716209cb4
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"0c9c193a-e44e-4b63-8762-b01716209cb4\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b351b993-9d63-4108-bd27-509f5c826a44\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"3dbeb3e2f19887c67eb495442c8bc20418933e3fb9a76f6263e0d8ff7f500bf1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"1c99104c-72e9-4c82-a3b0-5d7545e39019\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"01d3ed79cd8f6611ace1320414834ddecc40eea4210524827fc05ef963c989e4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13057, \"end_char_idx\": 16085, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "65 seconds after request: ,\n    Message received 1.65 seconds after request:  \n    Message received 1.67 seconds after request: 28\n    Message received 1.67 seconds after request: ,\n    Message received 1.67 seconds after request:  \n    Message received 1.69 seconds after request: 29\n    Message received 1.69 seconds after request: ,\n    Message received 1.69 seconds after request:  \n    Message received 1.80 seconds after request: 30\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 31\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 32\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 33\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 34\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 35\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.80 seconds after request: 36\n    Message received 1.80 seconds after request: ,\n    Message received 1.80 seconds after request:  \n    Message received 1.82 seconds after request: 37\n    Message received 1.82 seconds after request: ,\n    Message received 1.82 seconds after request:  \n    Message received 1.83 seconds after request: 38\n    Message received 1.83 seconds after request: ,\n    Message received 1.83 seconds after request:  \n    Message received 1.84 seconds after request: 39\n    Message received 1.84 seconds after request: ,\n    Message received 1.84 seconds after request:  \n    Message received 1.87 seconds after request: 40\n    Message received 1.87 seconds after request: ,\n    Message received 1.87 seconds after request:  \n    Message received 1.88 seconds after request: 41\n    Message received 1.88 seconds after request: ,\n    Message received 1.88 seconds after request:  \n    Message received 1.91 seconds after request: 42\n    Message received 1.91 seconds after request: ,\n    Message received 1.91 seconds after request:  \n    Message received 1.93 seconds after request: 43\n    Message received 1.93 seconds after request: ,\n    Message received 1.93 seconds after request:  \n    Message received 1.93 seconds after request: 44\n    Message received 1.93 seconds after request: ,\n    Message received 1.93 seconds after request:  \n    Message received 1.95 seconds after request: 45\n    Message received 1.95 seconds after request: ,\n    Message received 1.95 seconds after request:  \n    Message received 2.00 seconds after request: 46\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 1c99104c-72e9-4c82-a3b0-5d7545e39019
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"1c99104c-72e9-4c82-a3b0-5d7545e39019\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0c9c193a-e44e-4b63-8762-b01716209cb4\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"3f255cca9a7f77f8cfb8a701d8a6a2bbcfabba98e056f8f6a940f55d1df7445b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"4ab2a2d8-df76-47e5-8221-62bb7e56b28e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e373b8bb67795f0e105c05d36666c8abab17a55233444120239d2153e0956e03\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14545, \"end_char_idx\": 17574, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "82 seconds after request: 37\n    Message received 1.82 seconds after request: ,\n    Message received 1.82 seconds after request:  \n    Message received 1.83 seconds after request: 38\n    Message received 1.83 seconds after request: ,\n    Message received 1.83 seconds after request:  \n    Message received 1.84 seconds after request: 39\n    Message received 1.84 seconds after request: ,\n    Message received 1.84 seconds after request:  \n    Message received 1.87 seconds after request: 40\n    Message received 1.87 seconds after request: ,\n    Message received 1.87 seconds after request:  \n    Message received 1.88 seconds after request: 41\n    Message received 1.88 seconds after request: ,\n    Message received 1.88 seconds after request:  \n    Message received 1.91 seconds after request: 42\n    Message received 1.91 seconds after request: ,\n    Message received 1.91 seconds after request:  \n    Message received 1.93 seconds after request: 43\n    Message received 1.93 seconds after request: ,\n    Message received 1.93 seconds after request:  \n    Message received 1.93 seconds after request: 44\n    Message received 1.93 seconds after request: ,\n    Message received 1.93 seconds after request:  \n    Message received 1.95 seconds after request: 45\n    Message received 1.95 seconds after request: ,\n    Message received 1.95 seconds after request:  \n    Message received 2.00 seconds after request: 46\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 47\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 48\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 49\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 50\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 51\n    Message received 2.00 seconds after request: ,\n    Message received 2.04 seconds after request:  \n    Message received 2.04 seconds after request: 52\n    Message received 2.04 seconds after request: ,\n    Message received 2.04 seconds after request:  \n    Message received 2.04 seconds after request: 53\n    Message received 2.04 seconds after request: ,\n    Message received 2.13 seconds after request:  \n    Message received 2.13 seconds after request: 54\n    Message received 2.14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.14 seconds after request: 55\n    Message received 2.14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.14 seconds after request: 56\n    Message received 2.14 seconds after request: ,\n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 4ab2a2d8-df76-47e5-8221-62bb7e56b28e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"4ab2a2d8-df76-47e5-8221-62bb7e56b28e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"1c99104c-72e9-4c82-a3b0-5d7545e39019\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0e084c296705d592ffdb1a7d7ff87bd98bf72e6ed02509c8e967ba9e2a689fdf\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"c88893da-db43-485a-b0d7-739fc04c8078\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5d1551b475dd6cd0b5e3075c8b3260f5a1edfc05bee9458dfd05c39a041e25c3\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16015, \"end_char_idx\": 19063, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 47\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 48\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 49\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 50\n    Message received 2.00 seconds after request: ,\n    Message received 2.00 seconds after request:  \n    Message received 2.00 seconds after request: 51\n    Message received 2.00 seconds after request: ,\n    Message received 2.04 seconds after request:  \n    Message received 2.04 seconds after request: 52\n    Message received 2.04 seconds after request: ,\n    Message received 2.04 seconds after request:  \n    Message received 2.04 seconds after request: 53\n    Message received 2.04 seconds after request: ,\n    Message received 2.13 seconds after request:  \n    Message received 2.13 seconds after request: 54\n    Message received 2.14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.14 seconds after request: 55\n    Message received 2.14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.14 seconds after request: 56\n    Message received 2.14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.16 seconds after request: 57\n    Message received 2.16 seconds after request: ,\n    Message received 2.16 seconds after request:  \n    Message received 2.17 seconds after request: 58\n    Message received 2.17 seconds after request: ,\n    Message received 2.17 seconds after request:  \n    Message received 2.19 seconds after request: 59\n    Message received 2.19 seconds after request: ,\n    Message received 2.19 seconds after request:  \n    Message received 2.21 seconds after request: 60\n    Message received 2.21 seconds after request: ,\n    Message received 2.21 seconds after request:  \n    Message received 2.34 seconds after request: 61\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 62\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 63\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 64\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 65\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 66\n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: c88893da-db43-485a-b0d7-739fc04c8078
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"c88893da-db43-485a-b0d7-739fc04c8078\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"4ab2a2d8-df76-47e5-8221-62bb7e56b28e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"e6555779e7f71fd072de2d19f5306da8c0d5a3e5873645d358a046ac62babe32\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"42f67c89-1d94-49ee-a3d1-79886f30ee35\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"28a893365ef4430fa3782d06243d3d0f9bf42f202812d020076ad8c02d5ef537\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17523, \"end_char_idx\": 20551, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "14 seconds after request: ,\n    Message received 2.14 seconds after request:  \n    Message received 2.16 seconds after request: 57\n    Message received 2.16 seconds after request: ,\n    Message received 2.16 seconds after request:  \n    Message received 2.17 seconds after request: 58\n    Message received 2.17 seconds after request: ,\n    Message received 2.17 seconds after request:  \n    Message received 2.19 seconds after request: 59\n    Message received 2.19 seconds after request: ,\n    Message received 2.19 seconds after request:  \n    Message received 2.21 seconds after request: 60\n    Message received 2.21 seconds after request: ,\n    Message received 2.21 seconds after request:  \n    Message received 2.34 seconds after request: 61\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 62\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 63\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 64\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 65\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 66\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 67\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.36 seconds after request: 68\n    Message received 2.36 seconds after request: ,\n    Message received 2.36 seconds after request:  \n    Message received 2.36 seconds after request: 69\n    Message received 2.36 seconds after request: ,\n    Message received 2.36 seconds after request:  \n    Message received 2.38 seconds after request: 70\n    Message received 2.38 seconds after request: ,\n    Message received 2.38 seconds after request:  \n    Message received 2.39 seconds after request: 71\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 72\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 73\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 74\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 75\n    Message received 2.39 seconds after request: ,\n    Message received 2.40 seconds after request:  \n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 42f67c89-1d94-49ee-a3d1-79886f30ee35
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"42f67c89-1d94-49ee-a3d1-79886f30ee35\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c88893da-db43-485a-b0d7-739fc04c8078\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"8a9524853dcd9451e7d557b10818f5a466ce408b7c3ae19a5c768b9916101e74\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b056edea-e5f0-4fd7-8406-847b66c6ce8c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b636dfe2181dfdfeb0dbdd009acec1f21ac55c3f7718ac3892b0e4cc6137d3b6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 19011, \"end_char_idx\": 22040, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "34 seconds after request: 66\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.34 seconds after request: 67\n    Message received 2.34 seconds after request: ,\n    Message received 2.34 seconds after request:  \n    Message received 2.36 seconds after request: 68\n    Message received 2.36 seconds after request: ,\n    Message received 2.36 seconds after request:  \n    Message received 2.36 seconds after request: 69\n    Message received 2.36 seconds after request: ,\n    Message received 2.36 seconds after request:  \n    Message received 2.38 seconds after request: 70\n    Message received 2.38 seconds after request: ,\n    Message received 2.38 seconds after request:  \n    Message received 2.39 seconds after request: 71\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 72\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 73\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 74\n    Message received 2.39 seconds after request: ,\n    Message received 2.39 seconds after request:  \n    Message received 2.39 seconds after request: 75\n    Message received 2.39 seconds after request: ,\n    Message received 2.40 seconds after request:  \n    Message received 2.40 seconds after request: 76\n    Message received 2.40 seconds after request: ,\n    Message received 2.42 seconds after request:  \n    Message received 2.42 seconds after request: 77\n    Message received 2.42 seconds after request: ,\n    Message received 2.51 seconds after request:  \n    Message received 2.51 seconds after request: 78\n    Message received 2.51 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 79\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 80\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 81\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 82\n    Message received 2.52 seconds after request: ,\n    Message received 2.60 seconds after request:  \n    Message received 2.60 seconds after request: 83\n    Message received 2.60 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.64 seconds after request: 84\n    Message received 2.64 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.64 seconds after request: 85\n    Message received 2.64 seconds after request: ,\n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b056edea-e5f0-4fd7-8406-847b66c6ce8c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b056edea-e5f0-4fd7-8406-847b66c6ce8c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"42f67c89-1d94-49ee-a3d1-79886f30ee35\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"a989135aeba4face88b4e88d3e816052e28c5fa59dfdcfb232941f72ada0d799\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b6551eaa-ea15-4e4d-b9b9-ed16212e023a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b59df920697612fb222dc49f40cd05ff0f7b18da5e334c60757c57815ee3cb22\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20481, \"end_char_idx\": 23529, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "Message received 2.40 seconds after request:  \n    Message received 2.40 seconds after request: 76\n    Message received 2.40 seconds after request: ,\n    Message received 2.42 seconds after request:  \n    Message received 2.42 seconds after request: 77\n    Message received 2.42 seconds after request: ,\n    Message received 2.51 seconds after request:  \n    Message received 2.51 seconds after request: 78\n    Message received 2.51 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 79\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 80\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 81\n    Message received 2.52 seconds after request: ,\n    Message received 2.52 seconds after request:  \n    Message received 2.52 seconds after request: 82\n    Message received 2.52 seconds after request: ,\n    Message received 2.60 seconds after request:  \n    Message received 2.60 seconds after request: 83\n    Message received 2.60 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.64 seconds after request: 84\n    Message received 2.64 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.64 seconds after request: 85\n    Message received 2.64 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.66 seconds after request: 86\n    Message received 2.66 seconds after request: ,\n    Message received 2.66 seconds after request:  \n    Message received 2.66 seconds after request: 87\n    Message received 2.66 seconds after request: ,\n    Message received 2.66 seconds after request:  \n    Message received 2.68 seconds after request: 88\n    Message received 2.68 seconds after request: ,\n    Message received 2.68 seconds after request:  \n    Message received 2.69 seconds after request: 89\n    Message received 2.69 seconds after request: ,\n    Message received 2.69 seconds after request:  \n    Message received 2.72 seconds after request: 90\n    Message received 2.72 seconds after request: ,\n    Message received 2.72 seconds after request:  \n    Message received 2.82 seconds after request: 91\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 92\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 93\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 94\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 95\n    Message received 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b6551eaa-ea15-4e4d-b9b9-ed16212e023a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b6551eaa-ea15-4e4d-b9b9-ed16212e023a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b056edea-e5f0-4fd7-8406-847b66c6ce8c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d2ec27dd53ad670a2c23f3db08e89062880db1f48bfcdaf9c704e7b2b9285db3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f66fad73-155a-4bb3-bf46-04f8a35af49e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"4f939bed0b7e1404426396724df577d12c16802e12332af75cb6a4a63f66eb29\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21989, \"end_char_idx\": 24627, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "64 seconds after request: ,\n    Message received 2.64 seconds after request:  \n    Message received 2.66 seconds after request: 86\n    Message received 2.66 seconds after request: ,\n    Message received 2.66 seconds after request:  \n    Message received 2.66 seconds after request: 87\n    Message received 2.66 seconds after request: ,\n    Message received 2.66 seconds after request:  \n    Message received 2.68 seconds after request: 88\n    Message received 2.68 seconds after request: ,\n    Message received 2.68 seconds after request:  \n    Message received 2.69 seconds after request: 89\n    Message received 2.69 seconds after request: ,\n    Message received 2.69 seconds after request:  \n    Message received 2.72 seconds after request: 90\n    Message received 2.72 seconds after request: ,\n    Message received 2.72 seconds after request:  \n    Message received 2.82 seconds after request: 91\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 92\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 93\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 94\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 95\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 96\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 97\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 98\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 99\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 100\n    Message received 2.82 seconds after request: None\n    Full response received 2.82 seconds after request\n    Full conversation received: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55,"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: f66fad73-155a-4bb3-bf46-04f8a35af49e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"f66fad73-155a-4bb3-bf46-04f8a35af49e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b6551eaa-ea15-4e4d-b9b9-ed16212e023a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"36ac56e9b39402acdcd12d705d93dc75a30624ff2abcd86c82ed478586fbc290\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b9b343fb-31f6-41ad-8097-5e7656fcd7ae\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"80c3ef75ea4a379416a1aedbf3a57accd6de5353e65855ffc42c1de7208a8c74\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23477, \"end_char_idx\": 25846, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "82 seconds after request: 95\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 96\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 97\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 98\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 99\n    Message received 2.82 seconds after request: ,\n    Message received 2.82 seconds after request:  \n    Message received 2.82 seconds after request: 100\n    Message received 2.82 seconds after request: None\n    Full response received 2.82 seconds after request\n    Full conversation received: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100\n\n\n#### Time comparison\n\nIn the example above, both requests took about 4 to 5 seconds to fully complete. Request times will vary depending on load and other stochastic factors.\n\nHowever, with the streaming request, we received the first token after 0.1 seconds, and subsequent tokens every ~0.01-0.02 seconds.\n\n### 4. How to get token usage data for streamed chat completion response\n\nYou can get token usage statistics for your streamed response by setting `stream_options={\"include_usage\": True}`. When you do so, an extra chunk will be streamed as the final chunk. You can access the usage data for the entire request via the `usage` field on this chunk. A few important notes when you set `stream_options={\"include_usage\": True}`:\n* The value for the `usage` field on all chunks except for the last one will be null.\n* The `usage` field on the last chunk contains token usage statistics for the entire request.\n* The `choices` field on the last chunk will always be an empty array `[]`.\n\nLet's see how it works using the example in 2."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b9b343fb-31f6-41ad-8097-5e7656fcd7ae
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb", "title": "How to stream completions", "tokens": 7537, "retrieve_doc": true, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b9b343fb-31f6-41ad-8097-5e7656fcd7ae\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"3b7c77bc-a243-4713-890e-4afae30ee21e\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b617d332020ce29478029df8701ec2601f023e5fed5b3d394be06e7b9cc4bc57\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f66fad73-155a-4bb3-bf46-04f8a35af49e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/How_to_stream_completions.ipynb\", \"title\": \"How to stream completions\", \"tokens\": 7537, \"retrieve_doc\": true, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c4596fa6133a2419ae74d290692c765a6e85ec55ff6fde72c08d05b9be697db4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 24584, \"end_char_idx\": 27263, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "ref_doc_id": "3b7c77bc-a243-4713-890e-4afae30ee21e", "chroma:document": "45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100\n\n\n#### Time comparison\n\nIn the example above, both requests took about 4 to 5 seconds to fully complete. Request times will vary depending on load and other stochastic factors.\n\nHowever, with the streaming request, we received the first token after 0.1 seconds, and subsequent tokens every ~0.01-0.02 seconds.\n\n### 4. How to get token usage data for streamed chat completion response\n\nYou can get token usage statistics for your streamed response by setting `stream_options={\"include_usage\": True}`. When you do so, an extra chunk will be streamed as the final chunk. You can access the usage data for the entire request via the `usage` field on this chunk. A few important notes when you set `stream_options={\"include_usage\": True}`:\n* The value for the `usage` field on all chunks except for the last one will be null.\n* The `usage` field on the last chunk contains token usage statistics for the entire request.\n* The `choices` field on the last chunk will always be an empty array `[]`.\n\nLet's see how it works using the example in 2.\n\n\n```python\n# Example of an OpenAI ChatCompletion request with stream=True and stream_options={\"include_usage\": True}\n\n# a ChatCompletion request\nresponse = client.chat.completions.create(\n    model='gpt-4o-mini',\n    messages=[\n        {'role': 'user', 'content': \"What's 1+1? Answer in one word.\"}\n    ],\n    temperature=0,\n    stream=True,\n    stream_options={\"include_usage\": True}, # retrieving token usage for stream response\n)\n\nfor chunk in response:\n    print(f\"choices: {chunk.choices}\\nusage: {chunk.usage}\")\n    print(\"****************\")\n```\n\n    choices: [Choice(delta=ChoiceDelta(content='', function_call=None, role='assistant', tool_calls=None), finish_reason=None, index=0, logprobs=None)]\n    usage: None\n    ****************\n    choices: [Choice(delta=ChoiceDelta(content='Two', function_call=None, role=None, tool_calls=None), finish_reason=None, index=0, logprobs=None)]\n    usage: None\n    ****************\n    choices: [Choice(delta=ChoiceDelta(content='.', function_call=None, role=None, tool_calls=None), finish_reason=None, index=0, logprobs=None)]\n    usage: None\n    ****************\n    choices: [Choice(delta=ChoiceDelta(content=None, function_call=None, role=None, tool_calls=None), finish_reason='stop', index=0, logprobs=None)]\n    usage: None\n    ****************\n    choices: []\n    usage: CompletionUsage(completion_tokens=2, prompt_tokens=18, total_tokens=20)\n    ****************"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: c1996a2a-0ecd-44dd-a0d6-d1c1aa8a22bd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"c1996a2a-0ecd-44dd-a0d6-d1c1aa8a22bd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e5064503-5b4a-43a9-81ce-b35ed3d86727\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3ad3c2ce5822375dcc0da17307d60a2b6a9cc4717930c72646785072c48f8108\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3539, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "# Using logprobs for classification and Q&A evaluation\n\nThis notebook demonstrates the use of the `logprobs` parameter in the Chat Completions API. When `logprobs` is enabled, the API returns the log probabilities of each output token, along with a limited number of the most likely tokens at each token position and their log probabilities. The relevant request parameters are:\n* `logprobs`: Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message. This option is currently not available on the `gpt-4-vision-preview` model.\n* `top_logprobs`: An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.\n\nLog probabilities of output tokens indicate the likelihood of each token occurring in the sequence given the context. To simplify, a logprob is `log(p)`, where `p` = probability of a token occurring at a specific position based on the previous tokens in the context. Some key points about `logprobs`:\n* Higher log probabilities suggest a higher likelihood of the token in that context. This allows users to gauge the model's confidence in its output or explore alternative responses the model considered.\n* Logprob can be any negative number or `0.0`. `0.0` corresponds to 100% probability.\n* Logprobs allow us to compute the joint probability of a sequence as the sum of the logprobs of the individual tokens. This is useful for scoring and ranking model outputs. Another common approach is to take the average per-token logprob of a sentence to choose the best generation.\n* We can examine the `logprobs` assigned to different candidate tokens to understand what options the model considered plausible or implausible.\n\nWhile there are a wide array of use cases for `logprobs`, this notebook will focus on its use for:\n\n1. Classification tasks\n\n* Large Language Models excel at many classification tasks, but accurately measuring the model's confidence in its outputs can be challenging. `logprobs` provide a probability associated with each class prediction, enabling users to set their own classification or confidence thresholds.\n\n2. Retrieval (Q&A) evaluation\n\n* `logprobs` can assist with self-evaluation in retrieval applications. In the Q&A example, the model outputs a contrived `has_sufficient_context_for_answer` boolean, which can serve as a confidence score of whether the answer is contained in the retrieved content. Evaluations of this type can reduce retrieval-based hallucinations and enhance accuracy.\n\n3. Autocomplete\n*  `logprobs` could help us decide how to suggest words as a user is typing.\n\n4. Token highlighting and outputting bytes\n*  Users can easily create a token highlighter using the built in tokenization that comes with enabling `logprobs`. Additionally, the bytes parameter includes the ASCII encoding of each output character, which is particularly useful for reproducing emojis and special characters.\n\n5. Calculating perplexity\n* `logprobs` can be used to help us assess the model's overall confidence in a result and help us compare the confidence of results from different prompts.\n\n## 0. Imports and utils\n\n\n```python\nfrom openai import OpenAI\nfrom math import exp\nimport numpy as np\nfrom IPython.display import display, HTML\nimport os\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e5064503-5b4a-43a9-81ce-b35ed3d86727
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e5064503-5b4a-43a9-81ce-b35ed3d86727\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"c1996a2a-0ecd-44dd-a0d6-d1c1aa8a22bd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"963b9a365b6a761184c1f5d0c46949f402bd7a78ff1b2ed794d6209dd8161249\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9c8b8832-c1ca-4349-821c-f6ed53d4c132\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"024c34cf60ed7f8e2380259c61ac4e300d58c468a2b72b463ea1dea66c2680ac\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 3302, \"end_char_idx\": 6133, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\nfrom openai import OpenAI\nfrom math import exp\nimport numpy as np\nfrom IPython.display import display, HTML\nimport os\n\nclient = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\", \"<your OpenAI API key if not set as env var>\"))\n```\n\n\n```python\ndef get_completion(\n    messages: list[dict[str, str]],\n    model: str = \"gpt-4\",\n    max_tokens=500,\n    temperature=0,\n    stop=None,\n    seed=123,\n    tools=None,\n    logprobs=None,  # whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message..\n    top_logprobs=None,\n) -> str:\n    params = {\n        \"model\": model,\n        \"messages\": messages,\n        \"max_tokens\": max_tokens,\n        \"temperature\": temperature,\n        \"stop\": stop,\n        \"seed\": seed,\n        \"logprobs\": logprobs,\n        \"top_logprobs\": top_logprobs,\n    }\n    if tools:\n        params[\"tools\"] = tools\n\n    completion = client.chat.completions.create(**params)\n    return completion\n```\n\n## 1. Using `logprobs` to assess confidence for classification tasks\n\nLet's say we want to create a system to classify news articles into a set of pre-defined categories. Without `logprobs`, we can use Chat Completions to do this, but it is much more difficult to assess the certainty with which the model made its classifications.\n\nNow, with `logprobs` enabled, we can see exactly how confident the model is in its predictions, which is crucial for creating an accurate and trustworthy classifier. For example, if the log probability for the chosen category is high, this suggests the model is quite confident in its classification. If it's low, this suggests the model is less confident. This can be particularly useful in cases where the model's classification is not what you expected, or when the model's output needs to be reviewed or validated by a human.\n\nWe'll begin with a prompt that presents the model with four categories: **Technology, Politics, Sports, and Arts**. The model is then tasked with classifying articles into these categories based solely on their headlines.\n\n\n```python\nCLASSIFICATION_PROMPT = \"\"\"You will be given a headline of a news article.\nClassify the article into one of the following categories: Technology, Politics, Sports, and Art.\nReturn only the name of the category, and nothing else.\nMAKE SURE your output is one of the four categories stated.\nArticle headline: {headline}\"\"\"\n\n```\n\nLet's look at three sample headlines, and first begin with a standard Chat Completions output, without `logprobs`\n\n\n```python\nheadlines = [\n    \"Tech Giant Unveils Latest Smartphone Model with Advanced Photo-Editing Features.\",\n    \"Local Mayor Launches Initiative to Enhance Urban Public Transport.\",\n    \"Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\",\n]\n\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 9c8b8832-c1ca-4349-821c-f6ed53d4c132
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"9c8b8832-c1ca-4349-821c-f6ed53d4c132\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e5064503-5b4a-43a9-81ce-b35ed3d86727\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"207c09768243b69e734d7beb641467a957bae480a9d90f2fdeb3419de8f66ae4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"95f4203d-6d2a-463d-af0d-c5fb4b0cb9f6\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3b1b84bcabcb5a68c81422d9abc9862f320d16cb80b9873c5c2fead61c394c99\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5412, \"end_char_idx\": 8163, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\nCLASSIFICATION_PROMPT = \"\"\"You will be given a headline of a news article.\nClassify the article into one of the following categories: Technology, Politics, Sports, and Art.\nReturn only the name of the category, and nothing else.\nMAKE SURE your output is one of the four categories stated.\nArticle headline: {headline}\"\"\"\n\n```\n\nLet's look at three sample headlines, and first begin with a standard Chat Completions output, without `logprobs`\n\n\n```python\nheadlines = [\n    \"Tech Giant Unveils Latest Smartphone Model with Advanced Photo-Editing Features.\",\n    \"Local Mayor Launches Initiative to Enhance Urban Public Transport.\",\n    \"Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\",\n]\n\n```\n\n\n```python\nfor headline in headlines:\n    print(f\"\\nHeadline: {headline}\")\n    API_RESPONSE = get_completion(\n        [{\"role\": \"user\", \"content\": CLASSIFICATION_PROMPT.format(headline=headline)}],\n        model=\"gpt-4\",\n    )\n    print(f\"Category: {API_RESPONSE.choices[0].message.content}\\n\")\n```\n\n    \n    Headline: Tech Giant Unveils Latest Smartphone Model with Advanced Photo-Editing Features.\n    Category: Technology\n    \n    \n    Headline: Local Mayor Launches Initiative to Enhance Urban Public Transport.\n    Category: Politics\n    \n    \n    Headline: Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\n    Category: Art\n    \n\n\nHere we can see the selected category for each headline. However, we have no visibility into the confidence of the model in its predictions. Let's rerun the same prompt but with `logprobs` enabled, and `top_logprobs` set to 2 (this will show us the 2 most likely output tokens for each token). Additionally we can also output the linear probability of each output token, in order to convert the log probability to the more easily interprable scale of 0-100%. \n\n\n```python\nfor headline in headlines:\n    print(f\"\\nHeadline: {headline}\")\n    API_RESPONSE = get_completion(\n        [{\"role\": \"user\", \"content\": CLASSIFICATION_PROMPT.format(headline=headline)}],\n        model=\"gpt-4\",\n        logprobs=True,\n        top_logprobs=2,\n    )\n    top_two_logprobs = API_RESPONSE.choices[0].logprobs.content[0].top_logprobs\n    html_content = \"\"\n    for i, logprob in enumerate(top_two_logprobs, start=1):\n        html_content += (\n            f\"<span style='color: cyan'>Output token {i}:</span> {logprob.token}, \"\n            f\"<span style='color: darkorange'>logprobs:</span> {logprob.logprob}, \"\n            f\"<span style='color: magenta'>linear probability:</span> {np.round(np.exp(logprob.logprob)*100,2)}%<br>\"\n        )\n    display(HTML(html_content))\n    print(\"\\n\")\n```\n\n    \n    Headline: Tech Giant Unveils Latest Smartphone Model with Advanced Photo-Editing Features."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 95f4203d-6d2a-463d-af0d-c5fb4b0cb9f6
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"95f4203d-6d2a-463d-af0d-c5fb4b0cb9f6\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9c8b8832-c1ca-4349-821c-f6ed53d4c132\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"71ee9129058e40d1680fa8aa9dcfdc74c8be343cda439fcfe71ea95622aa5ef7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"0b023c0e-0ef9-409b-9c41-cb57cb5f629f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"179497ab64bc9f56d95e4bad86d779293e6b4a9eda260c441d23579e3bcabfc0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 6792, \"end_char_idx\": 9487, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Here we can see the selected category for each headline. However, we have no visibility into the confidence of the model in its predictions. Let's rerun the same prompt but with `logprobs` enabled, and `top_logprobs` set to 2 (this will show us the 2 most likely output tokens for each token). Additionally we can also output the linear probability of each output token, in order to convert the log probability to the more easily interprable scale of 0-100%. \n\n\n```python\nfor headline in headlines:\n    print(f\"\\nHeadline: {headline}\")\n    API_RESPONSE = get_completion(\n        [{\"role\": \"user\", \"content\": CLASSIFICATION_PROMPT.format(headline=headline)}],\n        model=\"gpt-4\",\n        logprobs=True,\n        top_logprobs=2,\n    )\n    top_two_logprobs = API_RESPONSE.choices[0].logprobs.content[0].top_logprobs\n    html_content = \"\"\n    for i, logprob in enumerate(top_two_logprobs, start=1):\n        html_content += (\n            f\"<span style='color: cyan'>Output token {i}:</span> {logprob.token}, \"\n            f\"<span style='color: darkorange'>logprobs:</span> {logprob.logprob}, \"\n            f\"<span style='color: magenta'>linear probability:</span> {np.round(np.exp(logprob.logprob)*100,2)}%<br>\"\n        )\n    display(HTML(html_content))\n    print(\"\\n\")\n```\n\n    \n    Headline: Tech Giant Unveils Latest Smartphone Model with Advanced Photo-Editing Features.\n\n\n\n<span style='color: cyan'>Output token 1:</span> Technology, <span style='color: darkorange'>logprobs:</span> -2.4584822e-06, <span style='color: magenta'>linear probability:</span> 100.0%<br><span style='color: cyan'>Output token 2:</span> Techn, <span style='color: darkorange'>logprobs:</span> -13.781253, <span style='color: magenta'>linear probability:</span> 0.0%<br>\n\n\n    \n    \n    \n    Headline: Local Mayor Launches Initiative to Enhance Urban Public Transport.\n\n\n\n<span style='color: cyan'>Output token 1:</span> Politics, <span style='color: darkorange'>logprobs:</span> -2.4584822e-06, <span style='color: magenta'>linear probability:</span> 100.0%<br><span style='color: cyan'>Output token 2:</span> Technology, <span style='color: darkorange'>logprobs:</span> -13.937503, <span style='color: magenta'>linear probability:</span> 0.0%<br>\n\n\n    \n    \n    \n    Headline: Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\n\n\n\n<span style='color: cyan'>Output token 1:</span> Art, <span style='color: darkorange'>logprobs:</span> -0.009169078, <span style='color: magenta'>linear probability:</span> 99.09%<br><span style='color: cyan'>Output token 2:</span> Sports, <span style='color: darkorange'>logprobs:</span> -4.696669, <span style='color: magenta'>linear probability:</span> 0.91%<br>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 0b023c0e-0ef9-409b-9c41-cb57cb5f629f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"0b023c0e-0ef9-409b-9c41-cb57cb5f629f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"95f4203d-6d2a-463d-af0d-c5fb4b0cb9f6\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"24076ef590de35e4c4115576ea4797eea0ae64450d44f84d494254cb728a10b5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e0788c8e-37bf-48df-bc9f-124bb1a23510\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f7b70d541e04fe42394d57c33feb604e7ed832b0d6448dcdd75c8e85ef828ad9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8562, \"end_char_idx\": 10722, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Headline: Local Mayor Launches Initiative to Enhance Urban Public Transport.\n\n\n\n<span style='color: cyan'>Output token 1:</span> Politics, <span style='color: darkorange'>logprobs:</span> -2.4584822e-06, <span style='color: magenta'>linear probability:</span> 100.0%<br><span style='color: cyan'>Output token 2:</span> Technology, <span style='color: darkorange'>logprobs:</span> -13.937503, <span style='color: magenta'>linear probability:</span> 0.0%<br>\n\n\n    \n    \n    \n    Headline: Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\n\n\n\n<span style='color: cyan'>Output token 1:</span> Art, <span style='color: darkorange'>logprobs:</span> -0.009169078, <span style='color: magenta'>linear probability:</span> 99.09%<br><span style='color: cyan'>Output token 2:</span> Sports, <span style='color: darkorange'>logprobs:</span> -4.696669, <span style='color: magenta'>linear probability:</span> 0.91%<br>\n\n\n    \n    \n\n\nAs expected from the first two headlines, `gpt-4` is nearly 100% confident in its classifications, as the content is clearly technology and politics focused respectively. However, the third headline combines both sports and art-related themes, so we see the model is less confident in its selection.\n\nThis shows how important using `logprobs` can be, as if we are using LLMs for classification tasks we can set confidence theshholds, or output several potential output tokens if the log probability of the selected output is not sufficiently high. For instance, if we are creating a recommendation engine to tag articles, we can automatically classify headlines crossing a certain threshold, and send the less certain headlines for manual review.\n\n## 2. Retrieval confidence scoring to reduce hallucinations\n\nTo reduce hallucinations, and the performance of our RAG-based Q&A system, we can use `logprobs` to evaluate how confident the model is in its retrieval.\n\nLet's say we have built a retrieval system using RAG for Q&A, but are struggling with hallucinated answers to our questions. *Note:* we will use a hardcoded article for this example, but see other entries in the cookbook for tutorials on using RAG for Q&A."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e0788c8e-37bf-48df-bc9f-124bb1a23510
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e0788c8e-37bf-48df-bc9f-124bb1a23510\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"0b023c0e-0ef9-409b-9c41-cb57cb5f629f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"89651e94531b6bf7e41f04e35b85fbd422e53ecc42172773a41fd7e7dba529d9\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e76f4265-0d8f-4b7b-ad58-77059b21bad3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f101fd8fea7509babdac6c13e1b9fe2f6bfde5bcba7cd251848b13cd3ab62a09\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 9040, \"end_char_idx\": 14366, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Headline: Tennis Champion Showcases Hidden Talents in Symphony Orchestra Debut\n\n\n\n<span style='color: cyan'>Output token 1:</span> Art, <span style='color: darkorange'>logprobs:</span> -0.009169078, <span style='color: magenta'>linear probability:</span> 99.09%<br><span style='color: cyan'>Output token 2:</span> Sports, <span style='color: darkorange'>logprobs:</span> -4.696669, <span style='color: magenta'>linear probability:</span> 0.91%<br>\n\n\n    \n    \n\n\nAs expected from the first two headlines, `gpt-4` is nearly 100% confident in its classifications, as the content is clearly technology and politics focused respectively. However, the third headline combines both sports and art-related themes, so we see the model is less confident in its selection.\n\nThis shows how important using `logprobs` can be, as if we are using LLMs for classification tasks we can set confidence theshholds, or output several potential output tokens if the log probability of the selected output is not sufficiently high. For instance, if we are creating a recommendation engine to tag articles, we can automatically classify headlines crossing a certain threshold, and send the less certain headlines for manual review.\n\n## 2. Retrieval confidence scoring to reduce hallucinations\n\nTo reduce hallucinations, and the performance of our RAG-based Q&A system, we can use `logprobs` to evaluate how confident the model is in its retrieval.\n\nLet's say we have built a retrieval system using RAG for Q&A, but are struggling with hallucinated answers to our questions. *Note:* we will use a hardcoded article for this example, but see other entries in the cookbook for tutorials on using RAG for Q&A.\n\n\n```python\n# Article retrieved\nada_lovelace_article = \"\"\"Augusta Ada King, Countess of Lovelace (n\u00e9e Byron; 10 December 1815 \u2013 27 November 1852) was an English mathematician and writer, chiefly known for her work on Charles Babbage's proposed mechanical general-purpose computer, the Analytical Engine. She was the first to recognise that the machine had applications beyond pure calculation.\nAda Byron was the only legitimate child of poet Lord Byron and reformer Lady Byron. All Lovelace's half-siblings, Lord Byron's other children, were born out of wedlock to other women. Byron separated from his wife a month after Ada was born and left England forever. He died in Greece when Ada was eight. Her mother was anxious about her upbringing and promoted Ada's interest in mathematics and logic in an effort to prevent her from developing her father's perceived insanity. Despite this, Ada remained interested in him, naming her two sons Byron and Gordon. Upon her death, she was buried next to him at her request. Although often ill in her childhood, Ada pursued her studies assiduously. She married William King in 1835. King was made Earl of Lovelace in 1838, Ada thereby becoming Countess of Lovelace.\nHer educational and social exploits brought her into contact with scientists such as Andrew Crosse, Charles Babbage, Sir David Brewster, Charles Wheatstone, Michael Faraday, and the author Charles Dickens, contacts which she used to further her education. Ada described her approach as \"poetical science\" and herself as an \"Analyst (& Metaphysician)\".\nWhen she was eighteen, her mathematical talents led her to a long working relationship and friendship with fellow British mathematician Charles Babbage, who is known as \"the father of computers\". She was in particular interested in Babbage's work on the Analytical Engine. Lovelace first met him in June 1833, through their mutual friend, and her private tutor, Mary Somerville.\nBetween 1842 and 1843, Ada translated an article by the military engineer Luigi Menabrea (later Prime Minister of Italy) about the Analytical Engine, supplementing it with an elaborate set of seven notes, simply called \"Notes\".\nLovelace's notes are important in the early history of computers, especially since the seventh one contained what many consider to be the first computer program\u2014that is, an algorithm designed to be carried out by a machine. Other historians reject this perspective and point out that Babbage's personal notes from the years 1836/1837 contain the first programs for the engine. She also developed a vision of the capability of computers to go beyond mere calculating or number-crunching, while many others, including Babbage himself, focused only on those capabilities. Her mindset of \"poetical science\" led her to ask questions about the Analytical Engine (as shown in her notes) examining how individuals and society relate to technology as a collaborative tool.\n\"\"\"\n\n# Questions that can be easily answered given the article\neasy_questions = [\n    \"What nationality was Ada Lovelace?\",\n    \"What was an important finding from Lovelace's seventh note?\",\n]\n\n# Questions that are not fully covered in the article\nmedium_questions = [\n    \"Did Lovelace collaborate with Charles Dickens\",\n    \"What concepts did Lovelace build with Charles Babbage\",\n]\n```\n\nNow, what we can do is ask the model to respond to the question, but then also evaluate its response. Specifically, we will ask the model to output a boolean `has_sufficient_context_for_answer`. We can then evaluate the `logprobs` to see just how confident the model is that its answer was contained in the provided context"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e76f4265-0d8f-4b7b-ad58-77059b21bad3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e76f4265-0d8f-4b7b-ad58-77059b21bad3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e0788c8e-37bf-48df-bc9f-124bb1a23510\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"faf55353813466824aa9706ac6b88bfd423de73974e636f764eecf244692e1e2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"45919a2d-0240-43a3-81ae-ce84a59b0f72\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d962395f75158624b821cd60a0b7c486fb99c8cdef821cd601b0307e73353bc9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14369, \"end_char_idx\": 16505, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\nPROMPT = \"\"\"You retrieved this article: {article}. The question is: {question}.\nBefore even answering the question, consider whether you have sufficient information in the article to answer the question fully.\nYour output should JUST be the boolean true or false, of if you have sufficient information in the article to answer the question.\nRespond with just one word, the boolean true or false. You must output the word 'True', or the word 'False', nothing else.\n\"\"\"\n\n```\n\n\n```python\nhtml_output = \"\"\nhtml_output += \"Questions clearly answered in article\"\n\nfor question in easy_questions:\n    API_RESPONSE = get_completion(\n        [\n            {\n                \"role\": \"user\",\n                \"content\": PROMPT.format(\n                    article=ada_lovelace_article, question=question\n                ),\n            }\n        ],\n        model=\"gpt-4\",\n        logprobs=True,\n    )\n    html_output += f'<p style=\"color:green\">Question: {question}</p>'\n    for logprob in API_RESPONSE.choices[0].logprobs.content:\n        html_output += f'<p style=\"color:cyan\">has_sufficient_context_for_answer: {logprob.token}, <span style=\"color:darkorange\">logprobs: {logprob.logprob}, <span style=\"color:magenta\">linear probability: {np.round(np.exp(logprob.logprob)*100,2)}%</span></p>'\n\nhtml_output += \"Questions only partially covered in the article\"\n\nfor question in medium_questions:\n    API_RESPONSE = get_completion(\n        [\n            {\n                \"role\": \"user\",\n                \"content\": PROMPT.format(\n                    article=ada_lovelace_article, question=question\n                ),\n            }\n        ],\n        model=\"gpt-4\",\n        logprobs=True,\n        top_logprobs=3,\n    )\n    html_output += f'<p style=\"color:green\">Question: {question}</p>'\n    for logprob in API_RESPONSE.choices[0].logprobs.content:\n        html_output += f'<p style=\"color:cyan\">has_sufficient_context_for_answer: {logprob.token}, <span style=\"color:darkorange\">logprobs: {logprob.logprob}, <span style=\"color:magenta\">linear probability: {np.round(np.exp(logprob.logprob)*100,2)}%</span></p>'\n\ndisplay(HTML(html_output))\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 45919a2d-0240-43a3-81ae-ce84a59b0f72
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"45919a2d-0240-43a3-81ae-ce84a59b0f72\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e76f4265-0d8f-4b7b-ad58-77059b21bad3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"6ce31c81369dd003f49a35dc6e29b56e4ae3799e430648448b54e81b9898dd4f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e3e196cc-2e6c-4304-94c4-123bb41ab272\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3b35b3c7ba36a28a7aee8e8375d4a560db7cf7c8935f4cc170895302eef33040\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 14854, \"end_char_idx\": 17663, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\nhtml_output = \"\"\nhtml_output += \"Questions clearly answered in article\"\n\nfor question in easy_questions:\n    API_RESPONSE = get_completion(\n        [\n            {\n                \"role\": \"user\",\n                \"content\": PROMPT.format(\n                    article=ada_lovelace_article, question=question\n                ),\n            }\n        ],\n        model=\"gpt-4\",\n        logprobs=True,\n    )\n    html_output += f'<p style=\"color:green\">Question: {question}</p>'\n    for logprob in API_RESPONSE.choices[0].logprobs.content:\n        html_output += f'<p style=\"color:cyan\">has_sufficient_context_for_answer: {logprob.token}, <span style=\"color:darkorange\">logprobs: {logprob.logprob}, <span style=\"color:magenta\">linear probability: {np.round(np.exp(logprob.logprob)*100,2)}%</span></p>'\n\nhtml_output += \"Questions only partially covered in the article\"\n\nfor question in medium_questions:\n    API_RESPONSE = get_completion(\n        [\n            {\n                \"role\": \"user\",\n                \"content\": PROMPT.format(\n                    article=ada_lovelace_article, question=question\n                ),\n            }\n        ],\n        model=\"gpt-4\",\n        logprobs=True,\n        top_logprobs=3,\n    )\n    html_output += f'<p style=\"color:green\">Question: {question}</p>'\n    for logprob in API_RESPONSE.choices[0].logprobs.content:\n        html_output += f'<p style=\"color:cyan\">has_sufficient_context_for_answer: {logprob.token}, <span style=\"color:darkorange\">logprobs: {logprob.logprob}, <span style=\"color:magenta\">linear probability: {np.round(np.exp(logprob.logprob)*100,2)}%</span></p>'\n\ndisplay(HTML(html_output))\n```\n\n\nQuestions clearly answered in article<p style=\"color:green\">Question: What nationality was Ada Lovelace?</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -3.1281633e-07, <span style=\"color:magenta\">linear probability: 100.0%</span></p><p style=\"color:green\">Question: What was an important finding from Lovelace's seventh note?</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -7.89631e-07, <span style=\"color:magenta\">linear probability: 100.0%</span></p>Questions only partially covered in the article<p style=\"color:green\">Question: Did Lovelace collaborate with Charles Dickens</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -0.06993677, <span style=\"color:magenta\">linear probability: 93.25%</span></p><p style=\"color:green\">Question: What concepts did Lovelace build with Charles Babbage</p><p style=\"color:cyan\">has_sufficient_context_for_answer: False, <span style=\"color:darkorange\">logprobs: -0.61807257, <span style=\"color:magenta\">linear probability: 53.9%</span></p>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e3e196cc-2e6c-4304-94c4-123bb41ab272
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e3e196cc-2e6c-4304-94c4-123bb41ab272\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"45919a2d-0240-43a3-81ae-ce84a59b0f72\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"70c489fe666f9cca537b384a457728c9a0c2aba92571b3d53b354bd38d07e7b3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"2cfbfceb-74ff-4a60-88dc-945ba5682f18\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"17eb89701ff59b70af19a91a422f91e6a599cd5b8339b4e30e002ac0769d8acb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16508, \"end_char_idx\": 19483, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Questions clearly answered in article<p style=\"color:green\">Question: What nationality was Ada Lovelace?</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -3.1281633e-07, <span style=\"color:magenta\">linear probability: 100.0%</span></p><p style=\"color:green\">Question: What was an important finding from Lovelace's seventh note?</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -7.89631e-07, <span style=\"color:magenta\">linear probability: 100.0%</span></p>Questions only partially covered in the article<p style=\"color:green\">Question: Did Lovelace collaborate with Charles Dickens</p><p style=\"color:cyan\">has_sufficient_context_for_answer: True, <span style=\"color:darkorange\">logprobs: -0.06993677, <span style=\"color:magenta\">linear probability: 93.25%</span></p><p style=\"color:green\">Question: What concepts did Lovelace build with Charles Babbage</p><p style=\"color:cyan\">has_sufficient_context_for_answer: False, <span style=\"color:darkorange\">logprobs: -0.61807257, <span style=\"color:magenta\">linear probability: 53.9%</span></p>\n\n\nFor the first two questions, our model asserts with (near) 100% confidence that the article has sufficient context to answer the posed questions.<br><br>\nOn the other hand, for the more tricky questions which are less clearly answered in the article, the model is less confident that it has sufficient context. This is a great guardrail to help ensure our retrieved content is sufficient.<br><br>\nThis self-evaluation can help reduce hallucinations, as you can restrict answers or re-prompt the user when your `sufficient_context_for_answer` log probability is below a certain threshold. Methods like this have been shown to significantly reduce RAG for Q&A hallucinations and errors ([Example](https://jfan001.medium.com/how-we-cut-the-rate-of-gpt-hallucinations-from-20-to-less-than-2-f3bfcc10e4ec)) \n\n\n```python\n\n```\n\n## 3. Autocomplete\n\nAnother use case for `logprobs` are autocomplete systems. Without creating the entire autocomplete system end-to-end, let's demonstrate how `logprobs` could help us decide how to suggest words as a user is typing.\n\nFirst, let's come up with a sample sentence: `\"My least favorite TV show is Breaking Bad.\"` Let's say we want it to dynamically recommend the next word or token as we are typing the sentence, but *only* if the model is quite sure of what the next word will be. To demonstrate this, let's break up the sentence into sequential components.\n\n\n```python\nsentence_list = [\n    \"My\",\n    \"My least\",\n    \"My least favorite\",\n    \"My least favorite TV\",\n    \"My least favorite TV show\",\n    \"My least favorite TV show is\",\n    \"My least favorite TV show is Breaking Bad\",\n]\n```\n\nNow, we can ask `gpt-3.5-turbo` to act as an autocomplete engine with whatever context the model is given. We can enable `logprobs` and can see how confident the model is in its prediction."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 2cfbfceb-74ff-4a60-88dc-945ba5682f18
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"2cfbfceb-74ff-4a60-88dc-945ba5682f18\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e3e196cc-2e6c-4304-94c4-123bb41ab272\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"3f565ca1d81e4a8e8a577eb4faa5856ab671762ecd41852003a0f0cfc9881bb6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"86269357-d6d5-4e49-abe6-7eba8a95907b\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3b2aced285b3fb80afce474dc95c633dae1c447e3bc2e38fbedf07ed4fd7eb88\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18471, \"end_char_idx\": 21345, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\n\n```\n\n## 3. Autocomplete\n\nAnother use case for `logprobs` are autocomplete systems. Without creating the entire autocomplete system end-to-end, let's demonstrate how `logprobs` could help us decide how to suggest words as a user is typing.\n\nFirst, let's come up with a sample sentence: `\"My least favorite TV show is Breaking Bad.\"` Let's say we want it to dynamically recommend the next word or token as we are typing the sentence, but *only* if the model is quite sure of what the next word will be. To demonstrate this, let's break up the sentence into sequential components.\n\n\n```python\nsentence_list = [\n    \"My\",\n    \"My least\",\n    \"My least favorite\",\n    \"My least favorite TV\",\n    \"My least favorite TV show\",\n    \"My least favorite TV show is\",\n    \"My least favorite TV show is Breaking Bad\",\n]\n```\n\nNow, we can ask `gpt-3.5-turbo` to act as an autocomplete engine with whatever context the model is given. We can enable `logprobs` and can see how confident the model is in its prediction.\n\n\n```python\nhigh_prob_completions = {}\nlow_prob_completions = {}\nhtml_output = \"\"\n\nfor sentence in sentence_list:\n    PROMPT = \"\"\"Complete this sentence. You are acting as auto-complete. Simply complete the sentence to the best of your ability, make sure it is just ONE sentence: {sentence}\"\"\"\n    API_RESPONSE = get_completion(\n        [{\"role\": \"user\", \"content\": PROMPT.format(sentence=sentence)}],\n        model=\"gpt-3.5-turbo\",\n        logprobs=True,\n        top_logprobs=3,\n    )\n    html_output += f'<p>Sentence: {sentence}</p>'\n    first_token = True\n    for token in API_RESPONSE.choices[0].logprobs.content[0].top_logprobs:\n        html_output += f'<p style=\"color:cyan\">Predicted next token: {token.token}, <span style=\"color:darkorange\">logprobs: {token.logprob}, <span style=\"color:magenta\">linear probability: {np.round(np.exp(token.logprob)*100,2)}%</span></p>'\n        if first_token:\n            if np.exp(token.logprob) > 0.95:\n                high_prob_completions[sentence] = token.token\n            if np.exp(token.logprob) < 0.60:\n                low_prob_completions[sentence] = token.token\n        first_token = False\n    html_output += \"<br>\"\n\ndisplay(HTML(html_output))\n```\n\n\n<p>Sentence: My</p><p style=\"color:cyan\">Predicted next token: favorite, <span style=\"color:darkorange\">logprobs: -0.18245785, <span style=\"color:magenta\">linear probability: 83.32%</span></p><p style=\"color:cyan\">Predicted next token: dog, <span style=\"color:darkorange\">logprobs: -2.397172, <span style=\"color:magenta\">linear probability: 9.1%</span></p><p style=\"color:cyan\">Predicted next token: ap, <span style=\"color:darkorange\">logprobs: -3.8732424, <span style=\"color:magenta\">linear probability: 2.08%</span></p><br><p>Sentence: My least</p><p style=\"color:cyan\">Predicted next token: favorite, <span style=\"color:darkorange\">logprobs: -0.0146376295,"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 86269357-d6d5-4e49-abe6-7eba8a95907b
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"86269357-d6d5-4e49-abe6-7eba8a95907b\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"2cfbfceb-74ff-4a60-88dc-945ba5682f18\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"23e1be41f644223f654013aedca105db5a4bc67332c8ea5e3850889725e13ad4\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b53ba361-1d73-4f55-94d7-b0f53b2589cf\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"489c28551420e75cb21109bd85577eebad9f785bf39d9f743e6c49aa2853e28c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20686, \"end_char_idx\": 23091, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "<p>Sentence: My</p><p style=\"color:cyan\">Predicted next token: favorite, <span style=\"color:darkorange\">logprobs: -0.18245785, <span style=\"color:magenta\">linear probability: 83.32%</span></p><p style=\"color:cyan\">Predicted next token: dog, <span style=\"color:darkorange\">logprobs: -2.397172, <span style=\"color:magenta\">linear probability: 9.1%</span></p><p style=\"color:cyan\">Predicted next token: ap, <span style=\"color:darkorange\">logprobs: -3.8732424, <span style=\"color:magenta\">linear probability: 2.08%</span></p><br><p>Sentence: My least</p><p style=\"color:cyan\">Predicted next token: favorite, <span style=\"color:darkorange\">logprobs: -0.0146376295, <span style=\"color:magenta\">linear probability: 98.55%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -4.2417912, <span style=\"color:magenta\">linear probability: 1.44%</span></p><p style=\"color:cyan\">Predicted next token:  favorite, <span style=\"color:darkorange\">logprobs: -9.748788, <span style=\"color:magenta\">linear probability: 0.01%</span></p><br><p>Sentence: My least favorite</p><p style=\"color:cyan\">Predicted next token: food, <span style=\"color:darkorange\">logprobs: -0.9481721, <span style=\"color:magenta\">linear probability: 38.74%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -1.3447137, <span style=\"color:magenta\">linear probability: 26.06%</span></p><p style=\"color:cyan\">Predicted next token: color, <span style=\"color:darkorange\">logprobs: -1.3887696, <span style=\"color:magenta\">linear probability: 24.94%</span></p><br><p>Sentence: My least favorite TV</p><p style=\"color:cyan\">Predicted next token: show, <span style=\"color:darkorange\">logprobs: -0.0007898556, <span style=\"color:magenta\">linear probability: 99.92%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -7.711523, <span style=\"color:magenta\">linear probability: 0.04%</span></p><p style=\"color:cyan\">Predicted next token: series, <span style=\"color:darkorange\">logprobs: -9.348547, <span style=\"color:magenta\">linear probability: 0.01%</span></p><br><p>Sentence: My least favorite TV show</p><p style=\"color:cyan\">Predicted next token: is, <span style=\"color:darkorange\">logprobs: -0.2851253, <span style=\"color:magenta\">linear probability: 75.19%</span></p><p style=\"color:cyan\">Predicted next token: of,"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b53ba361-1d73-4f55-94d7-b0f53b2589cf
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b53ba361-1d73-4f55-94d7-b0f53b2589cf\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"86269357-d6d5-4e49-abe6-7eba8a95907b\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"6f134d6dc2eb5582d2ad4ef30dbd65f3dd5471be2f300b656240cc3e8c4755e6\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"364cf924-fd23-452f-80b9-6fa58f4cb2db\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"18c3728606e296f31e9ba51726c3a99a654a83599580f7e023a8b8911119c86a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "9481721, <span style=\"color:magenta\">linear probability: 38.74%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -1.3447137, <span style=\"color:magenta\">linear probability: 26.06%</span></p><p style=\"color:cyan\">Predicted next token: color, <span style=\"color:darkorange\">logprobs: -1.3887696, <span style=\"color:magenta\">linear probability: 24.94%</span></p><br><p>Sentence: My least favorite TV</p><p style=\"color:cyan\">Predicted next token: show, <span style=\"color:darkorange\">logprobs: -0.0007898556, <span style=\"color:magenta\">linear probability: 99.92%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -7.711523, <span style=\"color:magenta\">linear probability: 0.04%</span></p><p style=\"color:cyan\">Predicted next token: series, <span style=\"color:darkorange\">logprobs: -9.348547, <span style=\"color:magenta\">linear probability: 0.01%</span></p><br><p>Sentence: My least favorite TV show</p><p style=\"color:cyan\">Predicted next token: is, <span style=\"color:darkorange\">logprobs: -0.2851253, <span style=\"color:magenta\">linear probability: 75.19%</span></p><p style=\"color:cyan\">Predicted next token: of, <span style=\"color:darkorange\">logprobs: -1.55335, <span style=\"color:magenta\">linear probability: 21.15%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -3.4928775, <span style=\"color:magenta\">linear probability: 3.04%</span></p><br><p>Sentence: My least favorite TV show is</p><p style=\"color:cyan\">Predicted next token: \"My, <span style=\"color:darkorange\">logprobs: -0.69349754, <span style=\"color:magenta\">linear probability: 49.98%</span></p><p style=\"color:cyan\">Predicted next token: \"The, <span style=\"color:darkorange\">logprobs: -1.2899293, <span style=\"color:magenta\">linear probability: 27.53%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -2.4170141, <span style=\"color:magenta\">linear probability: 8.92%</span></p><br><p>Sentence: My least favorite TV show is Breaking Bad</p><p style=\"color:cyan\">Predicted next token: because, <span style=\"color:darkorange\">logprobs: -0.17786823, <span style=\"color:magenta\">linear probability: 83.71%</span></p><p style=\"color:cyan\">Predicted next token: , <span style=\"color:darkorange\">logprobs: -2.3946173,"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 364cf924-fd23-452f-80b9-6fa58f4cb2db
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"364cf924-fd23-452f-80b9-6fa58f4cb2db\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b53ba361-1d73-4f55-94d7-b0f53b2589cf\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"3b8c664b2f32c10d2bff5caa56df79e32fb34a3726d2320f8bcd61676f03c7a7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"55212b9e-0037-4057-8a7c-c74e66111b5d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"83a39db323f2cd5410308436c5705c10bb39796273d4c59647b3164ecd485bdd\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "19%</span></p><p style=\"color:cyan\">Predicted next token: of, <span style=\"color:darkorange\">logprobs: -1.55335, <span style=\"color:magenta\">linear probability: 21.15%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -3.4928775, <span style=\"color:magenta\">linear probability: 3.04%</span></p><br><p>Sentence: My least favorite TV show is</p><p style=\"color:cyan\">Predicted next token: \"My, <span style=\"color:darkorange\">logprobs: -0.69349754, <span style=\"color:magenta\">linear probability: 49.98%</span></p><p style=\"color:cyan\">Predicted next token: \"The, <span style=\"color:darkorange\">logprobs: -1.2899293, <span style=\"color:magenta\">linear probability: 27.53%</span></p><p style=\"color:cyan\">Predicted next token: My, <span style=\"color:darkorange\">logprobs: -2.4170141, <span style=\"color:magenta\">linear probability: 8.92%</span></p><br><p>Sentence: My least favorite TV show is Breaking Bad</p><p style=\"color:cyan\">Predicted next token: because, <span style=\"color:darkorange\">logprobs: -0.17786823, <span style=\"color:magenta\">linear probability: 83.71%</span></p><p style=\"color:cyan\">Predicted next token: , <span style=\"color:darkorange\">logprobs: -2.3946173, <span style=\"color:magenta\">linear probability: 9.12%</span></p><p style=\"color:cyan\">Predicted next token: . <span style=\"color:darkorange\">logprobs: -3.1861975, <span style=\"color:magenta\">linear probability: 4.13%</span></p><br>\n\n\nLet's look at the high confidence autocompletions:\n\n\n```python\nhigh_prob_completions\n\n```\n\n\n\n\n    {'My least': 'favorite', 'My least favorite TV': 'show'}\n\n\n\nThese look reasonable! We can feel confident in those suggestions. It's pretty likely you want to write 'show' after writing 'My least favorite TV'! Now let's look at the autocompletion suggestions the model was less confident about:\n\n\n```python\nlow_prob_completions\n\n```\n\n\n\n\n    {'My least favorite': 'food', 'My least favorite TV show is': '\"My'}\n\n\n\nThese are logical as well. It's pretty unclear what the user is going to say with just the prefix 'my least favorite', and it's really anyone's guess what the author's favorite TV show is. <br><br>\nSo, using `gpt-3.5-turbo`, we can create the root of a dynamic autocompletion engine with `logprobs`!\n\n## 4. Highlighter and bytes parameter\n\nLet's quickly touch on creating a simple token highlighter with `logprobs`, and using the bytes parameter. First, we can create a function that counts and highlights each token. While this doesn't use the log probabilities, it uses the built in tokenization that comes with enabling `logprobs`."}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 55212b9e-0037-4057-8a7c-c74e66111b5d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"55212b9e-0037-4057-8a7c-c74e66111b5d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"364cf924-fd23-452f-80b9-6fa58f4cb2db\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"dfcfc1d4684fae02241e5f6bb237304ac5f11d9c694b1a5fc52b24f959928427\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6d3ba58f-4b7c-4f4b-8fd4-2a564fd32a77\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e85ee81c864e968eed4cd8227df8ea814e333dbab864ee74d5403b5476ddb5a0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": null, \"end_char_idx\": null, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "71%</span></p><p style=\"color:cyan\">Predicted next token: , <span style=\"color:darkorange\">logprobs: -2.3946173, <span style=\"color:magenta\">linear probability: 9.12%</span></p><p style=\"color:cyan\">Predicted next token: . <span style=\"color:darkorange\">logprobs: -3.1861975, <span style=\"color:magenta\">linear probability: 4.13%</span></p><br>\n\n\nLet's look at the high confidence autocompletions:\n\n\n```python\nhigh_prob_completions\n\n```\n\n\n\n\n    {'My least': 'favorite', 'My least favorite TV': 'show'}\n\n\n\nThese look reasonable! We can feel confident in those suggestions. It's pretty likely you want to write 'show' after writing 'My least favorite TV'! Now let's look at the autocompletion suggestions the model was less confident about:\n\n\n```python\nlow_prob_completions\n\n```\n\n\n\n\n    {'My least favorite': 'food', 'My least favorite TV show is': '\"My'}\n\n\n\nThese are logical as well. It's pretty unclear what the user is going to say with just the prefix 'my least favorite', and it's really anyone's guess what the author's favorite TV show is. <br><br>\nSo, using `gpt-3.5-turbo`, we can create the root of a dynamic autocompletion engine with `logprobs`!\n\n## 4. Highlighter and bytes parameter\n\nLet's quickly touch on creating a simple token highlighter with `logprobs`, and using the bytes parameter. First, we can create a function that counts and highlights each token. While this doesn't use the log probabilities, it uses the built in tokenization that comes with enabling `logprobs`.\n\n\n```python\nPROMPT = \"\"\"What's the longest word in the English language?\"\"\"\n\nAPI_RESPONSE = get_completion(\n    [{\"role\": \"user\", \"content\": PROMPT}], model=\"gpt-4\", logprobs=True, top_logprobs=5\n)\n\n\ndef highlight_text(api_response):\n    colors = [\n        \"#FF00FF\",  # Magenta\n        \"#008000\",  # Green\n        \"#FF8C00\",  # Dark Orange\n        \"#FF0000\",  # Red\n        \"#0000FF\",  # Blue\n    ]\n    tokens = api_response.choices[0].logprobs.content\n\n    color_idx = 0  # Initialize color index\n    html_output = \"\"  # Initialize HTML output\n    for t in tokens:\n        token_str = bytes(t.bytes).decode(\"utf-8\")  # Decode bytes to string\n\n        # Add colored token to HTML output\n        html_output += f\"<span style='color: {colors[color_idx]}'>{token_str}</span>\"\n\n        # Move to the next color\n        color_idx = (color_idx + 1) % len(colors)\n    display(HTML(html_output))  # Display HTML output\n    print(f\"Total number of tokens: {len(tokens)}\")\n```\n\n\n```python\nhighlight_text(API_RESPONSE)\n\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 6d3ba58f-4b7c-4f4b-8fd4-2a564fd32a77
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"6d3ba58f-4b7c-4f4b-8fd4-2a564fd32a77\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"55212b9e-0037-4057-8a7c-c74e66111b5d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"14c1ffc369932fb085a52f4bdd94723f41a8dd9acdbec51cae10e2a0bfdeb2eb\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b2b80524-36ac-4a6a-afa4-ee1d3e1c3dac\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a73e9e2da3d3a42a0e17676455ef58d16351d24e1891060c7e7557ecdb02b602\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 25639, \"end_char_idx\": 28690, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "```python\nPROMPT = \"\"\"What's the longest word in the English language?\"\"\"\n\nAPI_RESPONSE = get_completion(\n    [{\"role\": \"user\", \"content\": PROMPT}], model=\"gpt-4\", logprobs=True, top_logprobs=5\n)\n\n\ndef highlight_text(api_response):\n    colors = [\n        \"#FF00FF\",  # Magenta\n        \"#008000\",  # Green\n        \"#FF8C00\",  # Dark Orange\n        \"#FF0000\",  # Red\n        \"#0000FF\",  # Blue\n    ]\n    tokens = api_response.choices[0].logprobs.content\n\n    color_idx = 0  # Initialize color index\n    html_output = \"\"  # Initialize HTML output\n    for t in tokens:\n        token_str = bytes(t.bytes).decode(\"utf-8\")  # Decode bytes to string\n\n        # Add colored token to HTML output\n        html_output += f\"<span style='color: {colors[color_idx]}'>{token_str}</span>\"\n\n        # Move to the next color\n        color_idx = (color_idx + 1) % len(colors)\n    display(HTML(html_output))  # Display HTML output\n    print(f\"Total number of tokens: {len(tokens)}\")\n```\n\n\n```python\nhighlight_text(API_RESPONSE)\n\n```\n\n\n<span style='color: #FF00FF'>The</span><span style='color: #008000'> longest</span><span style='color: #FF8C00'> word</span><span style='color: #FF0000'> in</span><span style='color: #0000FF'> the</span><span style='color: #FF00FF'> English</span><span style='color: #008000'> language</span><span style='color: #FF8C00'>,</span><span style='color: #FF0000'> according</span><span style='color: #0000FF'> to</span><span style='color: #FF00FF'> the</span><span style='color: #008000'> Guinness</span><span style='color: #FF8C00'> World</span><span style='color: #FF0000'> Records</span><span style='color: #0000FF'>,</span><span style='color: #FF00FF'> is</span><span style='color: #008000'> '</span><span style='color: #FF8C00'>p</span><span style='color: #FF0000'>ne</span><span style='color: #0000FF'>um</span><span style='color: #FF00FF'>on</span><span style='color: #008000'>oul</span><span style='color: #FF8C00'>tram</span><span style='color: #FF0000'>icro</span><span style='color: #0000FF'>sc</span><span style='color: #FF00FF'>op</span><span style='color: #008000'>ics</span><span style='color: #FF8C00'>il</span><span style='color: #FF0000'>ic</span><span style='color: #0000FF'>ov</span><span style='color: #FF00FF'>ol</span><span style='color: #008000'>cano</span><span style='color: #FF8C00'>con</span><span style='color: #FF0000'>iosis</span><span style='color: #0000FF'>'.</span><span style='color: #FF00FF'> It</span><span style='color: #008000'> is</span><span style='color: #FF8C00'> a</span><span style='color: #FF0000'> type</span><span style='color: #0000FF'> of</span><span style='color: #FF00FF'> lung</span><span style='color: #008000'> disease</span><span style='color: #FF8C00'> caused</span><span style='color: #FF0000'> by</span><span style='color: #0000FF'> inh</span><span style='color: #FF00FF'>aling</span><span style='color: #008000'> ash</span><span style='color: #FF8C00'> and</span><span style='color: #FF0000'> sand</span><span style='color: #0000FF'> dust</span><span style='color: #FF00FF'>.</span>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b2b80524-36ac-4a6a-afa4-ee1d3e1c3dac
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b2b80524-36ac-4a6a-afa4-ee1d3e1c3dac\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6d3ba58f-4b7c-4f4b-8fd4-2a564fd32a77\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"36d675206e3a044988bf4aa835774e31bb531eb98bd16e5e2c06c26011c2561a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ba3cebe3-f234-4a2d-91f2-5d90a8009920\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d8916c94644d830c3cca220bf53190ac5f59d86eeb7ae9a84cea7c1fc6232e06\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 28697, \"end_char_idx\": 30663, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Total number of tokens: 51\n\n\nNext, let's reconstruct a sentence using the bytes parameter. With `logprobs` enabled, we are given both each token and the ASCII (decimal utf-8) values of the token string. These ASCII values can be helpful when handling tokens of or containing emojis or special characters.\n\n\n```python\nPROMPT = \"\"\"Output the blue heart emoji and its name.\"\"\"\nAPI_RESPONSE = get_completion(\n    [{\"role\": \"user\", \"content\": PROMPT}], model=\"gpt-4\", logprobs=True\n)\n\naggregated_bytes = []\njoint_logprob = 0.0\n\n# Iterate over tokens, aggregate bytes and calculate joint logprob\nfor token in API_RESPONSE.choices[0].logprobs.content:\n    print(\"Token:\", token.token)\n    print(\"Log prob:\", token.logprob)\n    print(\"Linear prob:\", np.round(exp(token.logprob) * 100, 2), \"%\")\n    print(\"Bytes:\", token.bytes, \"\\n\")\n    aggregated_bytes += token.bytes\n    joint_logprob += token.logprob\n\n# Decode the aggregated bytes to text\naggregated_text = bytes(aggregated_bytes).decode(\"utf-8\")\n\n# Assert that the decoded text is the same as the message content\nassert API_RESPONSE.choices[0].message.content == aggregated_text\n\n# Print the results\nprint(\"Bytes array:\", aggregated_bytes)\nprint(f\"Decoded bytes: {aggregated_text}\")\nprint(\"Joint prob:\", np.round(exp(joint_logprob) * 100, 2), \"%\")\n```\n\n    Token: \\xf0\\x9f\\x92\n    Log prob: -0.0003056686\n    Linear prob: 99.97 %\n    Bytes: [240, 159, 146] \n    \n    Token: \\x99\n    Log prob: 0.0\n    Linear prob: 100.0 %\n    Bytes: [153] \n    \n    Token:  -\n    Log prob: -0.0096905725\n    Linear prob: 99.04 %\n    Bytes: [32, 45] \n    \n    Token:  Blue\n    Log prob: -0.00042042506\n    Linear prob: 99.96 %\n    Bytes: [32, 66, 108, 117, 101] \n    \n    Token:  Heart\n    Log prob: -7.302705e-05\n    Linear prob: 99.99 %\n    Bytes: [32, 72, 101, 97, 114, 116] \n    \n    Bytes array: [240, 159, 146, 153, 32, 45, 32, 66, 108, 117, 101, 32, 72, 101, 97, 114, 116]\n    Decoded bytes: \ud83d\udc99 - Blue Heart\n    Joint prob: 98.96 %"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: ba3cebe3-f234-4a2d-91f2-5d90a8009920
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"ba3cebe3-f234-4a2d-91f2-5d90a8009920\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b2b80524-36ac-4a6a-afa4-ee1d3e1c3dac\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"00a7ce42477518bf9c6471b1dd1f567046f370a92a5dcb5a0bdaef830a158a12\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"28543fcd-3255-424e-8066-a45d9bb6a4db\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e8807722e9a48b46cc8898a7868543b8bbe73e871a9d60f5a46a9978f9f07271\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 30666, \"end_char_idx\": 32361, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Here, we see that while the first token was `\\xf0\\x9f\\x92'`, we can get its ASCII value and append it to a bytes array. Then, we can easily decode this array into a full sentence, and validate with our assert statement that the decoded bytes is the same as our completion message!\n\nAdditionally, we can get the joint probability of the entire completion, which is the exponentiated product of each token's log probability. This gives us how `likely` this given completion is given the prompt. Since, our prompt is quite directive (asking for a certain emoji and its name), the joint probability of this output is high! If we ask for a random output however, we'll see a much lower joint probability. This can also be a good tactic for developers during prompt engineering. \n\n## 5. Calculating perplexity\n\nWhen looking to assess the model's confidence in a result, it can be useful to calculate perplexity, which is a measure of the uncertainty. Perplexity can be calculated by exponentiating the negative of the average of the logprobs. Generally, a higher perplexity indicates a more uncertain result, and a lower perplexity indicates a more confident result. As such, perplexity can be used to both assess the result of an individual model run and also to compare the relative confidence of results between model runs. While a high confidence doesn't guarantee result accuracy, it can be a helpful signal that can be paired with other evaluation metrics to build a better understanding of your prompt's behavior.\n\nFor example, let's say that I want to use `gpt-3.5-turbo` to learn more about artificial intelligence. I could ask a question about recent history and a question about the future:"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 28543fcd-3255-424e-8066-a45d9bb6a4db
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"28543fcd-3255-424e-8066-a45d9bb6a4db\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ba3cebe3-f234-4a2d-91f2-5d90a8009920\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"047668dc27cfdaf5169ec8a4bfb7a1b842be984bdbcf7a7c510e47ccaada4b68\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6b5b21c7-010f-4492-a679-9bc556e643f0\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b67d0455b150122f6a0d07c6728044d9a289dc9d761d82dd202aa1f669cc94ff\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 30666, \"end_char_idx\": 35517, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "Here, we see that while the first token was `\\xf0\\x9f\\x92'`, we can get its ASCII value and append it to a bytes array. Then, we can easily decode this array into a full sentence, and validate with our assert statement that the decoded bytes is the same as our completion message!\n\nAdditionally, we can get the joint probability of the entire completion, which is the exponentiated product of each token's log probability. This gives us how `likely` this given completion is given the prompt. Since, our prompt is quite directive (asking for a certain emoji and its name), the joint probability of this output is high! If we ask for a random output however, we'll see a much lower joint probability. This can also be a good tactic for developers during prompt engineering. \n\n## 5. Calculating perplexity\n\nWhen looking to assess the model's confidence in a result, it can be useful to calculate perplexity, which is a measure of the uncertainty. Perplexity can be calculated by exponentiating the negative of the average of the logprobs. Generally, a higher perplexity indicates a more uncertain result, and a lower perplexity indicates a more confident result. As such, perplexity can be used to both assess the result of an individual model run and also to compare the relative confidence of results between model runs. While a high confidence doesn't guarantee result accuracy, it can be a helpful signal that can be paired with other evaluation metrics to build a better understanding of your prompt's behavior.\n\nFor example, let's say that I want to use `gpt-3.5-turbo` to learn more about artificial intelligence. I could ask a question about recent history and a question about the future:\n\n\n```python\nprompts = [\n    \"In a short sentence, has artifical intelligence grown in the last decade?\",\n    \"In a short sentence, what are your thoughts on the future of artificial intelligence?\",\n]\n\nfor prompt in prompts:\n    API_RESPONSE = get_completion(\n        [{\"role\": \"user\", \"content\": prompt}],\n        model=\"gpt-3.5-turbo\",\n        logprobs=True,\n    )\n\n    logprobs = [token.logprob for token in API_RESPONSE.choices[0].logprobs.content]\n    response_text = API_RESPONSE.choices[0].message.content\n    response_text_tokens = [token.token for token in API_RESPONSE.choices[0].logprobs.content]\n    max_starter_length = max(len(s) for s in [\"Prompt:\", \"Response:\", \"Tokens:\", \"Logprobs:\", \"Perplexity:\"])\n    max_token_length = max(len(s) for s in response_text_tokens)\n    \n\n    formatted_response_tokens = [s.rjust(max_token_length) for s in response_text_tokens]\n    formatted_lps = [f\"{lp:.2f}\".rjust(max_token_length) for lp in logprobs]\n\n    perplexity_score = np.exp(-np.mean(logprobs))\n    print(\"Prompt:\".ljust(max_starter_length), prompt)\n    print(\"Response:\".ljust(max_starter_length), response_text, \"\\n\")\n    print(\"Tokens:\".ljust(max_starter_length), \" \".join(formatted_response_tokens))\n    print(\"Logprobs:\".ljust(max_starter_length), \" \".join(formatted_lps))\n    print(\"Perplexity:\".ljust(max_starter_length), perplexity_score, \"\\n\")\n```\n\n    Prompt:     In a short sentence, has artifical intelligence grown in the last decade?\n    Response:   Yes, artificial intelligence has grown significantly in the last decade. \n    \n    Tokens:                Yes              ,     artificial   intelligence            has          grown  significantly             in            the           last         decade              .\n    Logprobs:            -0.00          -0.00          -0.00          -0.00          -0.00          -0.53          -0.11          -0.00          -0.00          -0.01          -0.00          -0.00\n    Perplexity: 1.0564125277713383 \n    \n    Prompt:     In a short sentence, what are your thoughts on the future of artificial intelligence?\n    Response:   The future of artificial intelligence holds great potential for transforming industries and improving efficiency, but also raises ethical and societal concerns that must be carefully addressed. \n    \n    Tokens:               The        future            of    artificial  intelligence         holds         great     potential           for  transforming    industries           and     improving    efficiency             ,           but          also        raises       ethical           and      societal      concerns          that          must            be     carefully     addressed             .\n    Logprobs:           -0.19         -0.03         -0.00         -0.00         -0.00         -0.30         -0.51         -0.24         -0.03         -1.45         -0.23         -0.03         -0.22         -0.83         -0.48         -0.01         -0.38         -0.07         -0.47         -0.63         -0.18         -0.26         -0.01         -0.14         -0.00         -0.59         -0.55         -0.00\n    Perplexity: 1.3220795252314004"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 6b5b21c7-010f-4492-a679-9bc556e643f0
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb", "title": "Using logprobs for classification and Q&A evaluation", "tokens": 9243, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"6b5b21c7-010f-4492-a679-9bc556e643f0\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"f503d917-ed02-45da-83f9-e74ba453d4de\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f314947ff47daa378d1fdb2f27fec83e4b52792457c0fcbdc406108ba67c95fe\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"28543fcd-3255-424e-8066-a45d9bb6a4db\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Using_logprobs.ipynb\", \"title\": \"Using logprobs for classification and Q&A evaluation\", \"tokens\": 9243, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"e6dcd8ab88126361f4ce86df5f35ae92c2e956ff8dba33304681496b7cedb98d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 35526, \"end_char_idx\": 36498, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "ref_doc_id": "f503d917-ed02-45da-83f9-e74ba453d4de", "chroma:document": "In this example, `gpt-3.5-turbo` returned a lower perplexity score for a more deterministic question about recent history, and a higher perplexity score for a more speculative assessment about the near future. Again, while these differences don't guarantee accuracy, they help point the way for our interpretation of the model's results and our future use of them.\n\n## 6. Conclusion\n\nNice! We were able to use the `logprobs` parameter to build a more robust classifier, evaluate our retrieval for Q&A system, and encode and decode each 'byte' of our tokens! `logprobs` adds useful information and signal to our completions output, and we are excited to see how developers incorporate it to improve applications.\n\n## 7. Possible extensions\n\nThere are many other use cases for `logprobs` that are not covered in this cookbook. We can use `logprobs` for:\n  - Moderation\n  - Keyword selection\n  - Improve prompts and interpretability of outputs\n  - Token healing\n  - and more!"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: a953a73d-33cb-44e3-a534-4e3da85036fd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"a953a73d-33cb-44e3-a534-4e3da85036fd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"d3eda7e1-f2f4-42bd-ae86-39e0b6695fb5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c7eeefa85011d7dd4302d92ec1c7760b942dcad18104a75b93834c92e0d27a97\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 0, \"end_char_idx\": 3027, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "# Parsing PDF documents for RAG applications\n\nThis notebook shows how to leverage GPT-4V to turn rich PDF documents such as slide decks or exports from web pages into usable content for your RAG application.\n\nThis technique can be used if you have a lot of unstructured data containing valuable information that you want to be able to retrieve as part of your RAG pipeline.\n\nFor example, you could build a Knowledge Assistant that could answer user queries about your company or product based on information contained in PDF documents. \n\nThe example documents used in this notebook are located at [data/example_pdfs](data/example_pdfs). They are related to OpenAI's APIs and various techniques that can be used as part of LLM projects.\n\n## Data preparation\n\nIn this section, we will process our input data to prepare it for retrieval.\n\nWe will do this in 2 ways:\n\n1. Extracting text with pdfminer\n2. Converting the PDF pages to images to analyze them with GPT-4V\n\nYou can skip the 1st method if you want to only use the content inferred from the image analysis.\n\n### Setup\n\nWe need to install a few libraries to convert the PDF to images and extract the text (optional).\n\n**Note: You need to install `poppler` on your machine for the `pdf2image` library to work. You can follow the instructions to install it [here](https://pypi.org/project/pdf2image/).**\n\n\n```python\n%pip install pdf2image\n%pip install pdfminer\n%pip install openai\n%pip install scikit-learn\n%pip install rich\n%pip install tqdm\n%pip install concurrent\n```\n\n\n```python\n# Imports\nfrom pdf2image import convert_from_path\nfrom pdf2image.exceptions import (\n    PDFInfoNotInstalledError,\n    PDFPageCountError,\n    PDFSyntaxError\n)\nfrom pdfminer.high_level import extract_text\nimport base64\nfrom io import BytesIO\nimport os\nimport concurrent\nfrom tqdm import tqdm\nfrom openai import OpenAI\nimport re\nimport pandas as pd \nfrom sklearn.metrics.pairwise import cosine_similarity\nimport json\nimport numpy as np\nfrom rich import print\nfrom ast import literal_eval\n```\n\n### File processing\n\n\n```python\ndef convert_doc_to_images(path):\n    images = convert_from_path(path)\n    return images\n\ndef extract_text_from_doc(path):\n    text = extract_text(path)\n    page_text = []\n    return text\n```\n\n#### Testing with an example\n\n\n```python\nfile_path = \"data/example_pdfs/fine-tuning-deck.pdf\"\n\nimages = convert_doc_to_images(file_path)\n```\n\n\n```python\ntext = extract_text_from_doc(file_path)\n```\n\n\n```python\nfor img in images:\n    display(img)\n```\n\n\n    \n![png](output_10_0.png)\n    \n\n\n\n    \n![png](output_10_1.png)\n    \n\n\n\n    \n![png](output_10_2.png)\n    \n\n\n\n    \n![png](output_10_3.png)\n    \n\n\n\n    \n![png](output_10_4.png)\n    \n\n\n\n    \n![png](output_10_5.png)\n    \n\n\n\n    \n![png](output_10_6.png)\n    \n\n\n### Image analysis with GPT-4V\n\nAfter converting a PDF file to multiple images, we'll use GPT-4V to analyze the content based on the images.\n\n\n```python\n# Initializing OpenAI client - see https://platform.openai.com/docs/quickstart?context=python\nclient = OpenAI()\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: d3eda7e1-f2f4-42bd-ae86-39e0b6695fb5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"d3eda7e1-f2f4-42bd-ae86-39e0b6695fb5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"a953a73d-33cb-44e3-a534-4e3da85036fd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"473e7fcae51fbd80c627de7d73e994cc85a692a7cedf5f67bf51983eec7427c3\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"96c58251-d9f8-4b0c-a627-43ab31208209\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5cfb408248c7d959a7df1682ee28cf59fa622df94eb22bbd1300654c288a9cc5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 1525, \"end_char_idx\": 3373, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\n# Imports\nfrom pdf2image import convert_from_path\nfrom pdf2image.exceptions import (\n    PDFInfoNotInstalledError,\n    PDFPageCountError,\n    PDFSyntaxError\n)\nfrom pdfminer.high_level import extract_text\nimport base64\nfrom io import BytesIO\nimport os\nimport concurrent\nfrom tqdm import tqdm\nfrom openai import OpenAI\nimport re\nimport pandas as pd \nfrom sklearn.metrics.pairwise import cosine_similarity\nimport json\nimport numpy as np\nfrom rich import print\nfrom ast import literal_eval\n```\n\n### File processing\n\n\n```python\ndef convert_doc_to_images(path):\n    images = convert_from_path(path)\n    return images\n\ndef extract_text_from_doc(path):\n    text = extract_text(path)\n    page_text = []\n    return text\n```\n\n#### Testing with an example\n\n\n```python\nfile_path = \"data/example_pdfs/fine-tuning-deck.pdf\"\n\nimages = convert_doc_to_images(file_path)\n```\n\n\n```python\ntext = extract_text_from_doc(file_path)\n```\n\n\n```python\nfor img in images:\n    display(img)\n```\n\n\n    \n![png](output_10_0.png)\n    \n\n\n\n    \n![png](output_10_1.png)\n    \n\n\n\n    \n![png](output_10_2.png)\n    \n\n\n\n    \n![png](output_10_3.png)\n    \n\n\n\n    \n![png](output_10_4.png)\n    \n\n\n\n    \n![png](output_10_5.png)\n    \n\n\n\n    \n![png](output_10_6.png)\n    \n\n\n### Image analysis with GPT-4V\n\nAfter converting a PDF file to multiple images, we'll use GPT-4V to analyze the content based on the images.\n\n\n```python\n# Initializing OpenAI client - see https://platform.openai.com/docs/quickstart?context=python\nclient = OpenAI()\n```\n\n\n```python\n# Converting images to base64 encoded images in a data URI format to use with the ChatCompletions API\ndef get_img_uri(img):\n    buffer = BytesIO()\n    img.save(buffer, format=\"jpeg\")\n    base64_image = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n    data_uri = f\"data:image/jpeg;base64,{base64_image}\"\n    return data_uri\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 96c58251-d9f8-4b0c-a627-43ab31208209
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"96c58251-d9f8-4b0c-a627-43ab31208209\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"d3eda7e1-f2f4-42bd-ae86-39e0b6695fb5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c71962f6b4b7bb6d025b8dcedd5a3ddd40708edb33400ad75bc058539728db1b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f37a7868-bbd9-4426-99c4-c0d42152a24d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6e6b018820e2f2be80becda6bbd93273637f2392d6ae23d9e464d881f68079dc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 2048, \"end_char_idx\": 5377, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\ndef convert_doc_to_images(path):\n    images = convert_from_path(path)\n    return images\n\ndef extract_text_from_doc(path):\n    text = extract_text(path)\n    page_text = []\n    return text\n```\n\n#### Testing with an example\n\n\n```python\nfile_path = \"data/example_pdfs/fine-tuning-deck.pdf\"\n\nimages = convert_doc_to_images(file_path)\n```\n\n\n```python\ntext = extract_text_from_doc(file_path)\n```\n\n\n```python\nfor img in images:\n    display(img)\n```\n\n\n    \n![png](output_10_0.png)\n    \n\n\n\n    \n![png](output_10_1.png)\n    \n\n\n\n    \n![png](output_10_2.png)\n    \n\n\n\n    \n![png](output_10_3.png)\n    \n\n\n\n    \n![png](output_10_4.png)\n    \n\n\n\n    \n![png](output_10_5.png)\n    \n\n\n\n    \n![png](output_10_6.png)\n    \n\n\n### Image analysis with GPT-4V\n\nAfter converting a PDF file to multiple images, we'll use GPT-4V to analyze the content based on the images.\n\n\n```python\n# Initializing OpenAI client - see https://platform.openai.com/docs/quickstart?context=python\nclient = OpenAI()\n```\n\n\n```python\n# Converting images to base64 encoded images in a data URI format to use with the ChatCompletions API\ndef get_img_uri(img):\n    buffer = BytesIO()\n    img.save(buffer, format=\"jpeg\")\n    base64_image = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n    data_uri = f\"data:image/jpeg;base64,{base64_image}\"\n    return data_uri\n```\n\n\n```python\nsystem_prompt = '''\nYou will be provided with an image of a pdf page or a slide. Your goal is to talk about the content that you see, in technical terms, as if you were delivering a presentation.\n\nIf there are diagrams, describe the diagrams and explain their meaning.\nFor example: if there is a diagram describing a process flow, say something like \"the process flow starts with X then we have Y and Z...\"\n\nIf there are tables, describe logically the content in the tables\nFor example: if there is a table listing items and prices, say something like \"the prices are the following: A for X, B for Y...\"\n\nDO NOT include terms referring to the content format\nDO NOT mention the content type - DO focus on the content itself\nFor example: if there is a diagram/chart and text on the image, talk about both without mentioning that one is a chart and the other is text.\nSimply describe what you see in the diagram and what you understand from the text.\n\nYou should keep it concise, but keep in mind your audience cannot see the image so be exhaustive in describing the content.\n\nExclude elements that are not relevant to the content:\nDO NOT mention page numbers or the position of the elements on the image.\n\n------\n\nIf there is an identifiable title, identify the title to give the output in the following format:\n\n{TITLE}\n\n{Content description}\n\nIf there is no clear title, simply return the content description.\n\n'''\n\ndef analyze_image(img_url):\n    response = client.chat.completions.create(\n    model=\"gpt-4-vision-preview\",\n    temperature=0,\n    messages=[\n        {\n            \"role\": \"system\",\n            \"content\": system_prompt\n        },\n        {\n            \"role\": \"user\",\n            \"content\": [\n                {\n                    \"type\": \"image_url\",\n                    \"image_url\": img_url,\n                },\n            ],\n        }\n    ],\n        max_tokens=300,\n        top_p=0.1\n    )\n\n    return response.choices[0].message.content\n```\n\n#### Testing with an example"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: f37a7868-bbd9-4426-99c4-c0d42152a24d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"f37a7868-bbd9-4426-99c4-c0d42152a24d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"96c58251-d9f8-4b0c-a627-43ab31208209\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"8b4ce093739f8218730b518e57e198fd2855a7a1c9f367eafb21dc918c44ff27\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"08e2bdb1-792c-42f2-ad36-f01605b1d952\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"6380550b509f50b7025d0eefcd638a3a1644c3d0eb7dc1393df8bab16d380beb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 5380, \"end_char_idx\": 8217, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\nimg = images[2]\ndata_uri = get_img_uri(img)\n```\n\n\n```python\nres = analyze_image(data_uri)\nprint(res)\n```\n\n    What is Fine-tuning\n    \n    Fine-tuning a model consists of training the model to follow a set of given input/output examples. This will teach the model to behave in a certain way when confronted with a similar input in the future.\n    \n    We recommend using 50-100 examples even if the minimum is 10.\n    \n    The process involves starting with a public model, using training data to train the model, and resulting in a fine-tuned model.\n\n\n#### Processing all documents\n\n\n```python\nfiles_path = \"data/example_pdfs\"\n\nall_items = os.listdir(files_path)\nfiles = [item for item in all_items if os.path.isfile(os.path.join(files_path, item))]\n```\n\n\n```python\ndef analyze_doc_image(img):\n    img_uri = get_img_uri(img)\n    data = analyze_image(img_uri)\n    return data\n```\n\nWe will list all files in the example folder and process them by \n1. Extracting the text\n2. Converting the docs to images\n3. Analyzing pages with GPT-4V\n\nNote: This takes about ~2 mins to run. Feel free to skip and load directly the result file (see below).\n\n\n```python\ndocs = []\n\nfor f in files:\n    \n    path = f\"{files_path}/{f}\"\n    doc = {\n        \"filename\": f\n    }\n    text = extract_text_from_doc(path)\n    doc['text'] = text\n    imgs = convert_doc_to_images(path)\n    pages_description = []\n    \n    print(f\"Analyzing pages for doc {f}\")\n    \n    # Concurrent execution\n    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:\n        \n        # Removing 1st slide as it's usually just an intro\n        futures = [\n            executor.submit(analyze_doc_image, img)\n            for img in imgs[1:]\n        ]\n        \n        with tqdm(total=len(imgs)-1) as pbar:\n            for _ in concurrent.futures.as_completed(futures):\n                pbar.update(1)\n        \n        for f in futures:\n            res = f.result()\n            pages_description.append(res)\n        \n    doc['pages_description'] = pages_description\n    docs.append(doc)\n```\n\n    Analyzing pages for doc rag-deck.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 19/19 [00:32<00:00,  1.72s/it]\n\n\n    Analyzing pages for doc models-page.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 9/9 [00:25<00:00,  2.80s/it]\n\n\n    Analyzing pages for doc evals-decks.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 12/12 [00:29<00:00,  2.44s/it]\n\n\n    Analyzing pages for doc fine-tuning-deck.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:19<00:00,  3.32s/it]\n\n\n\n```python\n# Saving result to file for later\njson_path = \"data/parsed_pdf_docs.json\"\n\nwith open(json_path, 'w') as f:\n    json.dump(docs, f)\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 08e2bdb1-792c-42f2-ad36-f01605b1d952
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"08e2bdb1-792c-42f2-ad36-f01605b1d952\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f37a7868-bbd9-4426-99c4-c0d42152a24d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"62225f2ed2783f9575b1fe0eeef336ef89b3401bc7d3e4e5e9437e2a0e03926a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"736dfb0a-3805-4e35-a491-71b316a22643\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"1448e142e45b970a5e8d1c4f77bb7f13d5dafa5f036151a89a93f56e412e0bd6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 7494, \"end_char_idx\": 10469, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 19/19 [00:32<00:00,  1.72s/it]\n\n\n    Analyzing pages for doc models-page.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 9/9 [00:25<00:00,  2.80s/it]\n\n\n    Analyzing pages for doc evals-decks.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 12/12 [00:29<00:00,  2.44s/it]\n\n\n    Analyzing pages for doc fine-tuning-deck.pdf\n\n\n    100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 6/6 [00:19<00:00,  3.32s/it]\n\n\n\n```python\n# Saving result to file for later\njson_path = \"data/parsed_pdf_docs.json\"\n\nwith open(json_path, 'w') as f:\n    json.dump(docs, f)\n```\n\n\n```python\n# Optional: load content from the saved file\nwith open(json_path, 'r') as f:\n    docs = json.load(f)\n```\n\n### Embedding content\nBefore embedding the content, we will chunk it logically by page.\nFor real-world scenarios, you could explore more advanced ways to chunk the content:\n- Cutting it into smaller pieces\n- Adding data - such as the slide title, deck title and/or the doc description - at the beginning of each piece of content. That way, each independent chunk can be in context\n\nFor the sake of brevity, we will use a very simple chunking strategy and rely on separators to split the text by page.\n\n\n```python\n# Chunking content by page and merging together slides text & description if applicable\ncontent = []\nfor doc in docs:\n    # Removing first slide as well\n    text = doc['text'].split('\\f')[1:]\n    description = doc['pages_description']\n    description_indexes = []\n    for i in range(len(text)):\n        slide_content = text[i] + '\\n'\n        # Trying to find matching slide description\n        slide_title = text[i].split('\\n')[0]\n        for j in range(len(description)):\n            description_title = description[j].split('\\n')[0]\n            if slide_title.lower() == description_title.lower():\n                slide_content += description[j].replace(description_title, '')\n                # Keeping track of the descriptions added\n                description_indexes.append(j)\n        # Adding the slide content + matching slide description to the content pieces\n        content.append(slide_content) \n    # Adding the slides descriptions that weren't used\n    for j in range(len(description)):\n        if j not in description_indexes:\n            content.append(description[j])\n```\n\n\n```python\nfor c in content:\n    print(c)\n    print(\"\\n\\n-------------------------------\\n\\n\")\n```\n\n\n```python\n# Cleaning up content\n# Removing trailing spaces, additional line breaks, page numbers and references to the content being a slide\nclean_content = []\nfor c in content:\n    text = c.replace(' \\n', '').replace('\\n\\n', '\\n').replace('\\n\\n\\n', '\\n').strip()\n    text = re.sub(r\"(?<=\\n)\\d{1,2}\", \"\", text)\n    text = re.sub(r\"\\b(?:the|this)\\s*slide\\s*\\w+\\b\", \"\", text, flags=re.IGNORECASE)\n    clean_content.append(text)\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 736dfb0a-3805-4e35-a491-71b316a22643
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"736dfb0a-3805-4e35-a491-71b316a22643\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"08e2bdb1-792c-42f2-ad36-f01605b1d952\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"9fdef58cfae9e23526cef920574e33670a0ddd006628c5dd078175f4fc4699f7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e8e1c055-d359-4d4e-941e-8e4eca4d6054\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5d213883a60134bfc6550b9a373135697b822225044e763240c4c6300b1f466b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 8839, \"end_char_idx\": 11748, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\n# Chunking content by page and merging together slides text & description if applicable\ncontent = []\nfor doc in docs:\n    # Removing first slide as well\n    text = doc['text'].split('\\f')[1:]\n    description = doc['pages_description']\n    description_indexes = []\n    for i in range(len(text)):\n        slide_content = text[i] + '\\n'\n        # Trying to find matching slide description\n        slide_title = text[i].split('\\n')[0]\n        for j in range(len(description)):\n            description_title = description[j].split('\\n')[0]\n            if slide_title.lower() == description_title.lower():\n                slide_content += description[j].replace(description_title, '')\n                # Keeping track of the descriptions added\n                description_indexes.append(j)\n        # Adding the slide content + matching slide description to the content pieces\n        content.append(slide_content) \n    # Adding the slides descriptions that weren't used\n    for j in range(len(description)):\n        if j not in description_indexes:\n            content.append(description[j])\n```\n\n\n```python\nfor c in content:\n    print(c)\n    print(\"\\n\\n-------------------------------\\n\\n\")\n```\n\n\n```python\n# Cleaning up content\n# Removing trailing spaces, additional line breaks, page numbers and references to the content being a slide\nclean_content = []\nfor c in content:\n    text = c.replace(' \\n', '').replace('\\n\\n', '\\n').replace('\\n\\n\\n', '\\n').strip()\n    text = re.sub(r\"(?<=\\n)\\d{1,2}\", \"\", text)\n    text = re.sub(r\"\\b(?:the|this)\\s*slide\\s*\\w+\\b\", \"\", text, flags=re.IGNORECASE)\n    clean_content.append(text)\n```\n\n\n```python\nfor c in clean_content:\n    print(c)\n    print(\"\\n\\n-------------------------------\\n\\n\")\n```\n\n\n```python\n# Creating the embeddings\n# We'll save to a csv file here for testing purposes but this is where you should load content in your vectorDB.\ndf = pd.DataFrame(clean_content, columns=['content'])\nprint(df.shape)\ndf.head()\n```\n\n    (64, 1)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Overview\\nRetrieval-Augmented Generationenhanc...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is RAG\\nRetrieve information to Augment t...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>When to use RAG\\nGood for  \u2705\\nNot good for  \u274c\\...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Technical patterns\\nData preparation\\nInput pr...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Technical patterns\\nData preparation\\nchunk do...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e8e1c055-d359-4d4e-941e-8e4eca4d6054
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e8e1c055-d359-4d4e-941e-8e4eca4d6054\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"736dfb0a-3805-4e35-a491-71b316a22643\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"7d445c9bdc2e4645a558e76f6f7e4fff1c1b0e17a44c0301fc4fbc004c3f7259\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"516d4777-57fe-453a-8814-d36fd4cec5a2\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"3186014fa5e279eac0d2e31db300695c60b087423678dec13e4c0e752fc067a1\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10472, \"end_char_idx\": 12106, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\nfor c in clean_content:\n    print(c)\n    print(\"\\n\\n-------------------------------\\n\\n\")\n```\n\n\n```python\n# Creating the embeddings\n# We'll save to a csv file here for testing purposes but this is where you should load content in your vectorDB.\ndf = pd.DataFrame(clean_content, columns=['content'])\nprint(df.shape)\ndf.head()\n```\n\n    (64, 1)\n\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Overview\\nRetrieval-Augmented Generationenhanc...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is RAG\\nRetrieve information to Augment t...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>When to use RAG\\nGood for  \u2705\\nNot good for  \u274c\\...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Technical patterns\\nData preparation\\nInput pr...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Technical patterns\\nData preparation\\nchunk do...</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\nembeddings_model = \"text-embedding-3-large\"\n\ndef get_embeddings(text):\n    embeddings = client.embeddings.create(\n      model=\"text-embedding-3-small\",\n      input=text,\n      encoding_format=\"float\"\n    )\n    return embeddings.data[0].embedding\n```\n\n\n```python\ndf['embeddings'] = df['content'].apply(lambda x: get_embeddings(x))\ndf.head()\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 516d4777-57fe-453a-8814-d36fd4cec5a2
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"516d4777-57fe-453a-8814-d36fd4cec5a2\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e8e1c055-d359-4d4e-941e-8e4eca4d6054\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d0a833dc928129834c6b7a1fffd6675b7a22e0f36ca03695f284e19933c1cc69\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9d332d9c-a890-455b-86fd-9f20af915d15\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"c79f64858aa4447d9f6d3e2547146f3f3af408a469a872b10271ce5767cce71b\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 10829, \"end_char_idx\": 13381, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Overview\\nRetrieval-Augmented Generationenhanc...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is RAG\\nRetrieve information to Augment t...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>When to use RAG\\nGood for  \u2705\\nNot good for  \u274c\\...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Technical patterns\\nData preparation\\nInput pr...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Technical patterns\\nData preparation\\nchunk do...</td>\n    </tr>\n  </tbody>\n</table>\n</div>\n\n\n\n\n```python\nembeddings_model = \"text-embedding-3-large\"\n\ndef get_embeddings(text):\n    embeddings = client.embeddings.create(\n      model=\"text-embedding-3-small\",\n      input=text,\n      encoding_format=\"float\"\n    )\n    return embeddings.data[0].embedding\n```\n\n\n```python\ndf['embeddings'] = df['content'].apply(lambda x: get_embeddings(x))\ndf.head()\n```\n\n\n\n\n<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>content</th>\n      <th>embeddings</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Overview\\nRetrieval-Augmented Generationenhanc...</td>\n      <td>[-0.014744381, 0.03017278, 0.06353764, 0.02110...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>What is RAG\\nRetrieve information to Augment t...</td>\n      <td>[-0.024337867, 0.022921458, -0.00971687, 0.010...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>When to use RAG\\nGood for  \u2705\\nNot good for  \u274c\\...</td>\n      <td>[-0.011084231, 0.021158217, -0.00430421, 0.017...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Technical patterns\\nData preparation\\nInput pr...</td>\n      <td>[-0.0058343858, 0.0408407, 0.054318383, 0.0190...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Technical patterns\\nData preparation\\nchunk do...</td>\n      <td>[-0.010359385, 0.03736894, 0.052995477, 0.0180...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 9d332d9c-a890-455b-86fd-9f20af915d15
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"9d332d9c-a890-455b-86fd-9f20af915d15\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"516d4777-57fe-453a-8814-d36fd4cec5a2\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"d5d3e15ae1f0f52a487704a3e692842c44155de88066c947fe7629a7dd393220\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"137ccf0b-17ef-4cc8-9cc5-d35e854f393c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"070101153f482981e2cf37960ce5d0056e0a0a1fb78c08ef2408d4c653157cee\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 13386, \"end_char_idx\": 16614, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\n# Saving locally for later\ndata_path = \"data/parsed_pdf_docs_with_embeddings.csv\"\ndf.to_csv(data_path, index=False)\n```\n\n\n```python\n# Optional: load data from saved file\ndf = pd.read_csv(data_path)\ndf[\"embeddings\"] = df.embeddings.apply(literal_eval).apply(np.array)\n```\n\n## Retrieval-augmented generation\n\nThe last step of the process is to generate outputs in response to input queries, after retrieving content as context to reply.\n\n\n```python\nsystem_prompt = '''\n    You will be provided with an input prompt and content as context that can be used to reply to the prompt.\n    \n    You will do 2 things:\n    \n    1. First, you will internally assess whether the content provided is relevant to reply to the input prompt. \n    \n    2a. If that is the case, answer directly using this content. If the content is relevant, use elements found in the content to craft a reply to the input prompt.\n\n    2b. If the content is not relevant, use your own knowledge to reply or say that you don't know how to respond if your knowledge is not sufficient to answer.\n    \n    Stay concise with your answer, replying specifically to the input prompt without mentioning additional information provided in the context content.\n'''\n\nmodel=\"gpt-4-turbo-preview\"\n\ndef search_content(df, input_text, top_k):\n    embedded_value = get_embeddings(input_text)\n    df[\"similarity\"] = df.embeddings.apply(lambda x: cosine_similarity(np.array(x).reshape(1,-1), np.array(embedded_value).reshape(1, -1)))\n    res = df.sort_values('similarity', ascending=False).head(top_k)\n    return res\n\ndef get_similarity(row):\n    similarity_score = row['similarity']\n    if isinstance(similarity_score, np.ndarray):\n        similarity_score = similarity_score[0][0]\n    return similarity_score\n\ndef generate_output(input_prompt, similar_content, threshold = 0.5):\n    \n    content = similar_content.iloc[0]['content']\n    \n    # Adding more matching content if the similarity is above threshold\n    if len(similar_content) > 1:\n        for i, row in similar_content.iterrows():\n            similarity_score = get_similarity(row)\n            if similarity_score > threshold:\n                content += f\"\\n\\n{row['content']}\"\n            \n    prompt = f\"INPUT PROMPT:\\n{input_prompt}\\n-------\\nCONTENT:\\n{content}\"\n    \n    completion = client.chat.completions.create(\n        model=model,\n        temperature=0.5,\n        messages=[\n            {\n                \"role\": \"system\",\n                \"content\": system_prompt\n            },\n            {\n                \"role\": \"user\",\n                \"content\": prompt\n            }\n        ]\n    )\n\n    return completion.choices[0].message.content\n```\n\n\n```python\n# Example user queries related to the content\nexample_inputs = [\n    'What are the main models you offer?',\n    'Do you have a speech recognition model?',\n    'Which embedding model should I use for non-English use cases?',\n    'Can I introduce new knowledge in my LLM app using RAG?',\n    'How many examples do I need to fine-tune a model?',\n    'Which metric can I use to evaluate a summarization task?',\n    'Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a ground truth.',\n]\n```"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 137ccf0b-17ef-4cc8-9cc5-d35e854f393c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"137ccf0b-17ef-4cc8-9cc5-d35e854f393c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9d332d9c-a890-455b-86fd-9f20af915d15\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"1250669e2cf1d4badf1a394ee42bf6ff2f0e9207672991895a1e99b6b0a57488\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"37bb8443-2d69-4a95-95f0-041dbff2c22f\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"803323320ba445ff42ded6ac759b8016c07198eac81676108b44bb86bc8b0943\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 16062, \"end_char_idx\": 18235, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "```python\n# Example user queries related to the content\nexample_inputs = [\n    'What are the main models you offer?',\n    'Do you have a speech recognition model?',\n    'Which embedding model should I use for non-English use cases?',\n    'Can I introduce new knowledge in my LLM app using RAG?',\n    'How many examples do I need to fine-tune a model?',\n    'Which metric can I use to evaluate a summarization task?',\n    'Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a ground truth.',\n]\n```\n\n\n```python\n# Running the RAG pipeline on each example\nfor ex in example_inputs:\n    print(f\"[deep_pink4][bold]QUERY:[/bold] {ex}[/deep_pink4]\\n\\n\")\n    matching_content = search_content(df, ex, 3)\n    print(f\"[grey37][b]Matching content:[/b][/grey37]\\n\")\n    for i, match in matching_content.iterrows():\n        print(f\"[grey37][i]Similarity: {get_similarity(match):.2f}[/i][/grey37]\")\n        print(f\"[grey37]{match['content'][:100]}{'...' if len(match['content']) > 100 else ''}[/[grey37]]\\n\\n\")\n    reply = generate_output(ex, matching_content)\n    print(f\"[turquoise4][b]REPLY:[/b][/turquoise4]\\n\\n[spring_green4]{reply}[/spring_green4]\\n\\n--------------\\n\\n\")\n```\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> What are the main models you offer?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 37bb8443-2d69-4a95-95f0-041dbff2c22f
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"37bb8443-2d69-4a95-95f0-041dbff2c22f\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"137ccf0b-17ef-4cc8-9cc5-d35e854f393c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"82fb908d2c5006acf6a3b9057a0a3205cf03c142394d86a1c016a63b4cdd5d60\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"edef560f-dee2-44bc-bff7-82d6321b90c5\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d9048fdececaa50e7bf31eb1efab831215f7983da756c843045a7f33112205b0\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 17287, \"end_char_idx\": 19292, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> What are the main models you offer?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content lists various API endpoints and their corresponding latest models:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">-...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.39</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: edef560f-dee2-44bc-bff7-82d6321b90c5
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"edef560f-dee2-44bc-bff7-82d6321b90c5\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"37bb8443-2d69-4a95-95f0-041dbff2c22f\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c86dd316fe08ca30281c5abe4d13408454029954fc3e2b674d4e36ececc31332\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"597f7517-6c2e-43e9-b8ee-490d6d2498a3\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a8d416c1cafe807a705cd70c37f48121dadf80f225fe176f8da6d68cfa99b4a2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 18240, \"end_char_idx\": 20467, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content lists various API endpoints and their corresponding latest models:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">-...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.39</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The Moderation models are designed to check whether content co...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 597f7517-6c2e-43e9-b8ee-490d6d2498a3
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"597f7517-6c2e-43e9-b8ee-490d6d2498a3\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"edef560f-dee2-44bc-bff7-82d6321b90c5\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"729b27389e17cb3f1a10dc7c6c4e729078d310d8815b512a3582f12911f9110e\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"006c53de-ff2f-4a85-a9e2-e24ddcd85a50\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ac905c9f39fad5867e873d1d1eb6a5f494a4c4c98f3eaa525cc3954e01529303\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 20470, \"end_char_idx\": 22866, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.39</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models provided by OpenAI, focusing on moderation models and GPT base ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">The main models we offer include:</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For completions: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-instruct, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For embeddings: text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-small, text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-large, and text-embedding-ada-</span><span style=\"color: #00875f; text-decoration-color: #00875f;"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 006c53de-ff2f-4a85-a9e2-e24ddcd85a50
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"006c53de-ff2f-4a85-a9e2-e24ddcd85a50\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"597f7517-6c2e-43e9-b8ee-490d6d2498a3\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f87c9a2882b534d18908dae832f36033f009f9528bf58ce0ab042324f207af36\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"8e6e9a94-2699-470b-8953-9c23b71ec80e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"7a094f28bf2fcaa6a66a30c57cb265a28a20b4ac815915c71ac8b1f4a317b489\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 21597, \"end_char_idx\": 24044, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">The main models we offer include:</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For completions: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-instruct, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For embeddings: text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-small, text-embedding-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-large, and text-embedding-ada-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For fine-tuning jobs: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For moderations: text-moderation-stable and text-moderation.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Additionally, we have the latest models like gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f;"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 8e6e9a94-2699-470b-8953-9c23b71ec80e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"8e6e9a94-2699-470b-8953-9c23b71ec80e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"006c53de-ff2f-4a85-a9e2-e24ddcd85a50\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"6584b30a0ebf41d73084fa67f7108ec5528baa16d14d3bd394b58e922a12ec20\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"272920f6-3106-44f3-85ac-f6ccc6ecc64a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"e5be8d308086f9a8a5de4522146e583bf1481aeaa0abd179ddca9daaef9a213c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 22835, \"end_char_idx\": 25163, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For fine-tuning jobs: gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo, babbage-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, and davinci-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">002</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">- For moderations: text-moderation-stable and text-moderation.</span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Additionally, we have the latest models like gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Do you have a speech recognition model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 272920f6-3106-44f3-85ac-f6ccc6ecc64a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"272920f6-3106-44f3-85ac-f6ccc6ecc64a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"8e6e9a94-2699-470b-8953-9c23b71ec80e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"aa6fe654abee4c46a5e3156f547016ff93ba6498a20adb8559555a0a828834cc\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"58b5e339-12a1-4cd2-9106-69f8b35f27da\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"de9f03a23d21939c8ee51742c77a36155acdc90fa6b6d8975f05fb7400d31c9c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 23877, \"end_char_idx\": 26084, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo-16k and fine-tuned versions of gpt-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3.5</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-turbo.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Do you have a speech recognition model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 58b5e339-12a1-4cd2-9106-69f8b35f27da
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"58b5e339-12a1-4cd2-9106-69f8b35f27da\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"272920f6-3106-44f3-85ac-f6ccc6ecc64a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"76b1dddf385429735ae52c05795b0dde1040a9646a2bf5f73b2358153f0eef89\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"917d79e5-2aa8-4419-9678-ebf714066cad\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a15ed8af85624a429687178daef5c2c1a08c97797cee34d9556a4c8c6941ee1d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 25168, \"end_char_idx\": 27705, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">MODEL</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DE S CRIPTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">tts-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">New  Text-to-speech </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The latest tex...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 917d79e5-2aa8-4419-9678-ebf714066cad
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"917d79e5-2aa8-4419-9678-ebf714066cad\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"58b5e339-12a1-4cd2-9106-69f8b35f27da\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"7c7302dfb6b74884a9e965a1bd6c9d85153191ee7a2420566e12edc860c9f91b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"91e35b5e-13dc-4665-aab5-ff9a6b347008\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"19fd98d37c86cb6136cacb44fe66023c9cf3b2a3ea0b9aea05098cae62526de6\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 27708, \"end_char_idx\": 29595, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.44</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ENDP OINT</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DATA USED</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">FOR TRAINING</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DEFAULT</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">RETENTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ELIGIBLE FO...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 91e35b5e-13dc-4665-aab5-ff9a6b347008
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"91e35b5e-13dc-4665-aab5-ff9a6b347008\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"917d79e5-2aa8-4419-9678-ebf714066cad\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b5d7d1962e406da35faf050bf5fdf35afe7e1c19ac3b97d3cc9556560ac39bd8\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"26d55122-c1d5-401d-9330-e31f9736c5c7\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d86cdf3a11a6bb310918e5aab51b22ba7ca96924281cb1c3f4cdf92b859f1cc7\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 29589, \"end_char_idx\": 32043, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, the Whisper model is a general-purpose speech recognition model mentioned in the content, capable of </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">multilingual speech recognition, speech translation, and language identification. The v2-large model, referred to </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">as </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"whisper-1\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\">, is available through an API and is optimized for faster performance.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which embedding model should I use for non-English use cases?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 26d55122-c1d5-401d-9330-e31f9736c5c7
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"26d55122-c1d5-401d-9330-e31f9736c5c7\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"91e35b5e-13dc-4665-aab5-ff9a6b347008\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c6421d35d80ef859b93278953e6c75ae5869200db0b2746e1735fc5d08d3ef9b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3d8a0b5e-4142-434c-895e-4c6d5d003d1c\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b5aff794b9108b65dde78bd3ca80ceda270860ff317ccbcde9269b5054f788bc\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 30865, \"end_char_idx\": 32400, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.46</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 3d8a0b5e-4142-434c-895e-4c6d5d003d1c
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"3d8a0b5e-4142-434c-895e-4c6d5d003d1c\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"26d55122-c1d5-401d-9330-e31f9736c5c7\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"3631a930bfc2aa925e450fda98ba4b6bfda505250015b282fdb688fe686c7a58\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ba8b8594-f92a-4a72-ae7f-deeb1b54b302\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"cad3e2b0b03ca2adbb162077da1a846065b401d6069eaae2694bc1c9b666ef12\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 31484, \"end_char_idx\": 34021, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The content describes various models related to text-to-speech, speech recognition, embeddings, and ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.46</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">MODEL</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">DE S CRIPTION</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">tts-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">New  Text-to-speech </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">1</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">The latest tex...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: ba8b8594-f92a-4a72-ae7f-deeb1b54b302
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"ba8b8594-f92a-4a72-ae7f-deeb1b54b302\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3d8a0b5e-4142-434c-895e-4c6d5d003d1c\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f39e31f519d34260c13b39a5cbafdb97520b5e0bf23b2473c3a3cee9d0a7a8c7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e72a43e2-9c15-408e-ae01-e0c3348c2a95\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"0aac3eae59addae4e703accba947b48b023bc827ac42c7e85ffe75a09d41d92c\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 34024, \"end_char_idx\": 35794, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.40</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">26</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">02</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">2024</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">, </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">17:58</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Models - OpenAI API</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Multilingual capabilities</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">GPT-</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">4</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\"> outperforms both previous larg...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e72a43e2-9c15-408e-ae01-e0c3348c2a95
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e72a43e2-9c15-408e-ae01-e0c3348c2a95\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ba8b8594-f92a-4a72-ae7f-deeb1b54b302\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"c9dfc1c53e7bdcfaca38d910367db028d756101d3c81341094727ce8f1c03c95\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"cb7cedfc-d828-4e3b-9994-101810012921\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"aa9077f13241d95863161de656c028f4d1a2e0aebe3671457fac577b59bde19e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 35788, \"end_char_idx\": 37795, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">For non-English use cases, you should use the </span><span style=\"color: #00875f; text-decoration-color: #00875f\">\"V3 large\"</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> embedding model, as it is described as the most capable </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">for both English and non-English tasks, with an output dimension of </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">3</span><span style=\"color: #00875f; text-decoration-color: #00875f\">,</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">072</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Can I introduce new knowledge in my LLM app using RAG?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: cb7cedfc-d828-4e3b-9994-101810012921
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"cb7cedfc-d828-4e3b-9994-101810012921\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e72a43e2-9c15-408e-ae01-e0c3348c2a95\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"99dc4096f0b877b24ed065a0b4c57195e98af28ea68ee8145a968e8932740da5\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"e36cf28d-f852-41d1-8eb6-93c6db8e6b2d\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"9b2f55dd8c2c238cf83b7a7520048ec39b972a8e3633846abd80328cfa5196e9\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 36817, \"end_char_idx\": 38852, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Can I introduce new knowledge in my LLM app using RAG?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Retrieve information to Augment the model\u2019s knowledge and Generate the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u201cWhat is y...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: e36cf28d-f852-41d1-8eb6-93c6db8e6b2d
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"e36cf28d-f852-41d1-8eb6-93c6db8e6b2d\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"cb7cedfc-d828-4e3b-9994-101810012921\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b3245080a02b4c3b4d33a0c073030cf2838a33cea5b5fe3c68b987e21fa1272a\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"3e085b01-14c8-4cd5-9ab7-8d71c72e6b75\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"f9eb8763ba491b4ae4c80a7b6363ff234a6dffa7a5f2de3d1857b5f5814db18a\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 37800, \"end_char_idx\": 39892, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Retrieve information to Augment the model\u2019s knowledge and Generate the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u201cWhat is y...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to use RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Good for  \u2705</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Not good for  \u274c</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Introducing new information to the model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Teaching ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 3e085b01-14c8-4cd5-9ab7-8d71c72e6b75
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"3e085b01-14c8-4cd5-9ab7-8d71c72e6b75\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"e36cf28d-f852-41d1-8eb6-93c6db8e6b2d\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"5335b46fbc5e212c1ec10a5eb52d664fa3903da2889a0fe716254988c0feec41\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"6e641843-1d7f-417e-a3a8-74c6fba40338\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b79afdf59a8853970e405a3d57258838df60ebabbe0c5c0d2f763efa99370f8e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 38857, \"end_char_idx\": 41017, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to use RAG</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Good for  \u2705</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Not good for  \u274c</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Introducing new information to the model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Teaching ...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Data preparation: augmenting content</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What does \u201cAugmentingcontent\u201d mean?</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Augmenti...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 6e641843-1d7f-417e-a3a8-74c6fba40338
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"6e641843-1d7f-417e-a3a8-74c6fba40338\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"3e085b01-14c8-4cd5-9ab7-8d71c72e6b75\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"213541a5bef9d3eeef15c601bea3c2c786450e94899bd489b5d69bc3956754c7\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"52779bbc-d4e7-4990-aef4-776522f6192a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"a37011a59e753a8bd76b2629edb02a625e2ecb35b443836a67aeeb82905146af\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 39886, \"end_char_idx\": 42238, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.43</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Data preparation: augmenting content</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What does \u201cAugmentingcontent\u201d mean?</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Augmenti...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> How many examples do I need to fine-tune a model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 52779bbc-d4e7-4990-aef4-776522f6192a
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"52779bbc-d4e7-4990-aef4-776522f6192a\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"6e641843-1d7f-417e-a3a8-74c6fba40338\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"bbcbcb6a4466d7fbbb386cf5c28e00c48e5306cc59ba1e56b4b193a083bd011d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"aad38009-3d55-4ee2-856d-68d6dfb26005\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"914aac184da76168151c5f196379d55ca66060490f4b231b01ea31cb7426e2f4\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 41011, \"end_char_idx\": 43499, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">Yes, you can introduce new knowledge in your LLM app using RAG by retrieving information from a knowledge base or </span>\n<span style=\"color: #00875f; text-decoration-color: #00875f\">external sources to augment the model's knowledge and generate outputs relevant to the queries posed.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> How many examples do I need to fine-tune a model?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.68</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is Fine-tuning</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Public Model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training data</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tunedmodel</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning a model consists...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: aad38009-3d55-4ee2-856d-68d6dfb26005
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"aad38009-3d55-4ee2-856d-68d6dfb26005\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"52779bbc-d4e7-4990-aef4-776522f6192a\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"5014e2031e7c8693cbadd44f2b9beebe56ed778913e3f71b4e2496d3b29ebf56\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"052512d5-6068-4217-a23d-31fcc3950e5e\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5f6fe07088436f8c23e3b357fde7ec514c72b794553450952763bc7d44ed6c4e\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 42600, \"end_char_idx\": 44624, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What is Fine-tuning</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Public Model</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training data</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Training</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tunedmodel</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning a model consists...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.62</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to fine-tune</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning is good for:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Following a given format or tone for the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Proce...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 052512d5-6068-4217-a23d-31fcc3950e5e
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"052512d5-6068-4217-a23d-31fcc3950e5e\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"aad38009-3d55-4ee2-856d-68d6dfb26005\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"363ab8556764113ba6a9b3cf834b8c661c821ed88509b730eb7c6eeb4f0c5a2d\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"fce1d88d-b47d-4a4b-8b60-9bfe11def6cd\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"813a2099892ca6bd73f0843192ae159e6c4a25eab17eb08d9e60716b336f19f2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 43493, \"end_char_idx\": 45613, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.62</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">When to fine-tune</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning is good for:</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Following a given format or tone for the output</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">- Proce...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Overview</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning involves adjusting theparameters of pre-trained models on aspeci\ufb01c dataset or t...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: fce1d88d-b47d-4a4b-8b60-9bfe11def6cd
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"fce1d88d-b47d-4a4b-8b60-9bfe11def6cd\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"052512d5-6068-4217-a23d-31fcc3950e5e\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"9c5f1019b49f1822fb10a306e629f8c1a74a574105fd26ee9a8d7371ba4c115b\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b77d1b5a-00d1-4a58-9cae-6e5fe7e7f2a1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"8bc9fa8ce34c4b65204984a98a574fe7924cdc583107ab9f09cf33e9683bf5f5\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 44618, \"end_char_idx\": 47112, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.57</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Overview</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Fine-tuning involves adjusting theparameters of pre-trained models on aspeci\ufb01c dataset or t...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">We recommend using </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">50</span><span style=\"color: #00875f; text-decoration-color: #00875f\">-</span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">100</span><span style=\"color: #00875f; text-decoration-color: #00875f\"> examples for fine-tuning a model, even though the minimum is </span><span style=\"color: #00875f; text-decoration-color: #00875f; font-weight: bold\">10</span><span style=\"color: #00875f; text-decoration-color: #00875f\">.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which metric can I use to evaluate a summarization task?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b77d1b5a-00d1-4a58-9cae-6e5fe7e7f2a1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b77d1b5a-00d1-4a58-9cae-6e5fe7e7f2a1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"fce1d88d-b47d-4a4b-8b60-9bfe11def6cd\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"63fb60496ffb44f8e08f1d4d2b868438dad9919a8bca90250ce207e778f99786\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"ed0ae047-4211-4c98-aa06-6dae19a05ea1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"23b24f5646d126a18d73f7e318fad42e04cd46b401b6a23d47e75eb3067331cb\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 46489, \"end_char_idx\": 48526, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Which metric can I use to evaluate a summarization task?</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.53</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ROUGE is a common metric for evaluating machine summariz...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: ed0ae047-4211-4c98-aa06-6dae19a05ea1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"ed0ae047-4211-4c98-aa06-6dae19a05ea1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"b77d1b5a-00d1-4a58-9cae-6e5fe7e7f2a1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"24c2e83d384ad6f7d127427051c6d431e9d9bb1154a8b3a1df2a0d59a47eceb2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"7b0e4313-6308-451f-baf3-3963eee64d63\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"37be058a33be894c3ed59e93a7b5323b801b43e93f6f91c8b87d23bcfda110df\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 47474, \"end_char_idx\": 49498, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">ROUGE is a common metric for evaluating machine summariz...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.49</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Component evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Subjective evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Compari...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 7b0e4313-6308-451f-baf3-3963eee64d63
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"7b0e4313-6308-451f-baf3-3963eee64d63\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"ed0ae047-4211-4c98-aa06-6dae19a05ea1\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"ef7fad780fe6eb5a79159e4a384b2f6c417da0ad215b585bc73507ecd6268a1f\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"9b7b149f-3cfc-41d1-9150-84f330cdf6ca\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"b7b0386d7119179de071a83afe5d53fc654116e61ea983d1f66847886f79e47d\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 48531, \"end_char_idx\": 50555, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Component evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Subjective evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Compari...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.48</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">BLEU score is another standard metric, this time focusin...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 9b7b149f-3cfc-41d1-9150-84f330cdf6ca
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"9b7b149f-3cfc-41d1-9150-84f330cdf6ca\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"7b0e4313-6308-451f-baf3-3963eee64d63\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f63b7bc768b8e2f26e5af98fb8edf43bd431f0708378246da710cebb10ac015c\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"af113ea2-5342-4b78-96dc-2f1b7fbb5f48\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"d64ff46d20f88f1726d61f508953d8c11f8b54905fa9d997c4ea17018821e4c2\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 49492, \"end_char_idx\": 51703, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.48</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">BLEU score is another standard metric, this time focusin...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">ROUGE is a common metric you can use to evaluate a summarization task.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>\n<span style=\"color: #af005f; text-decoration-color: #af005f\">ground truth.</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: af113ea2-5342-4b78-96dc-2f1b7fbb5f48
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"af113ea2-5342-4b78-96dc-2f1b7fbb5f48\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"9b7b149f-3cfc-41d1-9150-84f330cdf6ca\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"40ef5047b37a9e02a74c6cea8157ad2d72b25f5258f4525bf65e8f7936c750b0\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"f5a802d5-f330-4f89-b66d-5ef6ed7fa261\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"21ae55db8ab4fe75d8d4c228c3166894aceaca747fd5523c99f1f54151f08420\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 50549, \"end_char_idx\": 52828, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008787; text-decoration-color: #008787; font-weight: bold\">REPLY:</span>\n\n<span style=\"color: #00875f; text-decoration-color: #00875f\">ROUGE is a common metric you can use to evaluate a summarization task.</span>\n\n--------------\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #af005f; text-decoration-color: #af005f; font-weight: bold\">QUERY:</span><span style=\"color: #af005f; text-decoration-color: #af005f\"> Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a</span>\n<span style=\"color: #af005f; text-decoration-color: #af005f\">ground truth.</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">Matching content:</span>\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.60</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Our ground truth matches the predicted answer, so the evaluation passes!</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Eval...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: f5a802d5-f330-4f89-b66d-5ef6ed7fa261
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"f5a802d5-f330-4f89-b66d-5ef6ed7fa261\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"af113ea2-5342-4b78-96dc-2f1b7fbb5f48\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"f5dfb3d58595a110f427641fb2d4310a2eb5fae16338cdcec127015844f4c6e1\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"60ebf99b-f6f6-45b7-abd3-a585e0f1a269\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"390f2232ed3c77cd6474f0ef027e27972a09866bb451bc2ceade6168bd1fd7fa\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 51708, \"end_char_idx\": 53885, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.60</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Our ground truth matches the predicted answer, so the evaluation passes!</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Eval...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.59</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: 60ebf99b-f6f6-45b7-abd3-a585e0f1a269
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"60ebf99b-f6f6-45b7-abd3-a585e0f1a269\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"f5a802d5-f330-4f89-b66d-5ef6ed7fa261\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"b50b02272eabcf899393e855bccad4eb09d438db9a66174aad4635f111fc54ea\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"b4633451-b834-4d23-81c5-145b06e4c0b1\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"ee47e9d7e2d52f10d301ada39b25dee618d51c492b9037c5df1b728a76863c70\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 52822, \"end_char_idx\": 54242, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.59</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>"}


created_at: 2024-07-28 17:45:51
topic: persistent://default/default/308e7593-7e45-4080-bc68-ad4f88ce5b4b
id: b4633451-b834-4d23-81c5-145b06e4c0b1
encoding: FLOAT32
metadata:
{"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb", "title": "Parsing PDF documents for RAG applications", "tokens": 17715, "retrieve_doc": false, "source": "OpenAI_Cookbooks", "_node_content": "{\"id_\": \"b4633451-b834-4d23-81c5-145b06e4c0b1\", \"embedding\": null, \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"excluded_embed_metadata_keys\": [\"url\", \"tokens\", \"retrieve_doc\", \"source\"], \"excluded_llm_metadata_keys\": [\"title\", \"tokens\", \"retrieve_doc\", \"source\"], \"relationships\": {\"1\": {\"node_id\": \"6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f\", \"node_type\": \"4\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"0dfce4284c29148170f2773c9757bb61d730eadd741d4da27f0bb06eb617c262\", \"class_name\": \"RelatedNodeInfo\"}, \"2\": {\"node_id\": \"60ebf99b-f6f6-45b7-abd3-a585e0f1a269\", \"node_type\": \"1\", \"metadata\": {\"url\": \"https://github.com/openai/openai-cookbook/blob/main/examples/Parse_PDF_docs_for_RAG.ipynb\", \"title\": \"Parsing PDF documents for RAG applications\", \"tokens\": 17715, \"retrieve_doc\": false, \"source\": \"OpenAI_Cookbooks\"}, \"hash\": \"9c65e6e64cfa3fddff40575d2928fda39a4cbe8ec98b4560c8687797a786c2a2\", \"class_name\": \"RelatedNodeInfo\"}, \"3\": {\"node_id\": \"21755151-9747-4f4d-aaa8-6f3a2140765a\", \"node_type\": \"1\", \"metadata\": {}, \"hash\": \"5e2796b03cc5e9a22146d789ca69b93e8a03a016d867cad6ab570ddb10c24641\", \"class_name\": \"RelatedNodeInfo\"}}, \"text\": \"\", \"mimetype\": \"text/plain\", \"start_char_idx\": 53190, \"end_char_idx\": 55214, \"text_template\": \"{metadata_str}\\n\\n{content}\", \"metadata_template\": \"{key}: {value}\", \"metadata_seperator\": \"\\n\", \"class_name\": \"TextNode\"}", "_node_type": "TextNode", "document_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "ref_doc_id": "6ca68ea7-724d-4fe9-80e8-3c74fbd2c09f", "chroma:document": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What are evals</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Example</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">An evaluation contains a question and a correct answer. We call this the grou...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-style: italic\">Similarity: </span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold; font-style: italic\">0.50</span>\n</pre>\n\n\n\n\n<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Technical patterns</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">Metric-based evaluations</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What they\u2019re good for</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">What to be aware of</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">\u25cf</span>\n<span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">A good sta...</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">[</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f\">/</span><span style=\"color: #5f5f5f; text-decoration-color: #5f5f5f; font-weight: bold\">]</span>\n\n\n</pre>"}


