# embedding_metadata

key: chroma:document
string_value:
480 / 869
    481 / 869
    482 / 869
    483 / 869
    484 / 869
    485 / 869
    486 / 869
    487 / 869
    488 / 869
    489 / 869
    490 / 869
    491 / 869
    492 / 869
    493 / 869
    494 / 869
    495 / 869
    496 / 869
    497 / 869
    498 / 869
    499 / 869
    500 / 869
    501 / 869
    502 / 869
    503 / 869
    504 / 869
    505 / 869
    506 / 869
    507 / 869
    508 / 869
    509 / 869
    510 / 869
    511 / 869
    512 / 869
    513 / 869
    514 / 869
    515 / 869
    516 / 869
    517 / 869
    518 / 869
    519 / 869
    520 / 869
    521 / 869
    522 / 869
    523 / 869
    524 / 869
    525 / 869
    526 / 869
    527 / 869
    528 / 869
    529 / 869
    530 / 869
    531 / 869
    532 / 869
    533 / 869
    534 / 869
    535 / 869
    536 / 869
    537 / 869
    538 / 869
    539 / 869
    540 / 869
    541 / 869
    542 / 869
    543 / 869
    544 / 869
    545 / 869
    546 / 869
    547 / 869
    548 / 869
    549 / 869
    550 / 869
    551 / 869
    552 / 869
    553 / 869
    554 / 869
    555 / 869
    556 / 869
    557 / 869
    558 / 869
    559 / 869
    560 / 869
    561 / 869
    562 / 869
    563 / 869
    564 / 869
    565 / 869


key: _node_content
string_value:
{"id_": "29d86ab0-5cb4-4dd3-abda-bf533897dad8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a0e3bde7-6311-498a-9ae3-54561d774107", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "24e8cd3a848b4aad666db9d9fd6033e524e4d0fb3585547912f1c6266e48fc6e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4ad19158-10e7-48ac-9ab2-6da0d82d6885", "node_type": "1", "metadata": {}, "hash": "3aef619707a5e237aae88f166547aa28e65b7f14ae9e3f88b88086cf3640d69a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14590, "end_char_idx": 15793, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
/ 869
    522 / 869
    523 / 869
    524 / 869
    525 / 869
    526 / 869
    527 / 869
    528 / 869
    529 / 869
    530 / 869
    531 / 869
    532 / 869
    533 / 869
    534 / 869
    535 / 869
    536 / 869
    537 / 869
    538 / 869
    539 / 869
    540 / 869
    541 / 869
    542 / 869
    543 / 869
    544 / 869
    545 / 869
    546 / 869
    547 / 869
    548 / 869
    549 / 869
    550 / 869
    551 / 869
    552 / 869
    553 / 869
    554 / 869
    555 / 869
    556 / 869
    557 / 869
    558 / 869
    559 / 869
    560 / 869
    561 / 869
    562 / 869
    563 / 869
    564 / 869
    565 / 869
    566 / 869
    567 / 869
    568 / 869
    569 / 869
    570 / 869
    571 / 869
    572 / 869
    573 / 869
    574 / 869
    575 / 869
    576 / 869
    577 / 869
    578 / 869
    579 / 869
    580 / 869
    581 / 869
    582 / 869
    583 / 869
    584 / 869
    585 / 869
    586 / 869
    587 / 869
    588 / 869
    589 / 869
    590 / 869
    591 / 869
    592 / 869
    593 / 869
    594 / 869
    595 / 869
    596 / 869
    597 / 869
    598 / 869
    599 / 869
    600 / 869
    601 / 869
    602 / 869
    603 / 869
    604 / 869
    605 / 869
    606 / 869
    607


key: _node_content
string_value:
{"id_": "4ad19158-10e7-48ac-9ab2-6da0d82d6885", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "29d86ab0-5cb4-4dd3-abda-bf533897dad8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "29097cb2430559fd85bc1e756a644f929251cda471ad3d0c96c941e8bb581c21", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b1bc8604-0550-4df9-9ff7-ea04dc35f997", "node_type": "1", "metadata": {}, "hash": "4eb7dc84d1966049a89dbd277efbc9735543079e342e3e7697f46ce9beb00042", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15174, "end_char_idx": 16373, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
563 / 869
    564 / 869
    565 / 869
    566 / 869
    567 / 869
    568 / 869
    569 / 869
    570 / 869
    571 / 869
    572 / 869
    573 / 869
    574 / 869
    575 / 869
    576 / 869
    577 / 869
    578 / 869
    579 / 869
    580 / 869
    581 / 869
    582 / 869
    583 / 869
    584 / 869
    585 / 869
    586 / 869
    587 / 869
    588 / 869
    589 / 869
    590 / 869
    591 / 869
    592 / 869
    593 / 869
    594 / 869
    595 / 869
    596 / 869
    597 / 869
    598 / 869
    599 / 869
    600 / 869
    601 / 869
    602 / 869
    603 / 869
    604 / 869
    605 / 869
    606 / 869
    607 / 869
    608 / 869
    609 / 869
    610 / 869
    611 / 869
    612 / 869
    613 / 869
    614 / 869
    615 / 869
    616 / 869
    617 / 869
    618 / 869
    619 / 869
    620 / 869
    621 / 869
    622 / 869
    623 / 869
    624 / 869
    625 / 869
    626 / 869
    627 / 869
    628 / 869
    629 / 869
    630 / 869
    631 / 869
    632 / 869
    633 / 869
    634 / 869
    635 / 869
    636 / 869
    637 / 869
    638 / 869
    639 / 869
    640 / 869
    641 / 869
    642 / 869
    643 / 869
    644 / 869
    645 / 869
    646 / 869
    647 / 869
    648 / 869


key: _node_content
string_value:
{"id_": "b1bc8604-0550-4df9-9ff7-ea04dc35f997", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4ad19158-10e7-48ac-9ab2-6da0d82d6885", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f8966e9d2b94cc8a9ad1c42bd4826537f1a1ae74906df419ed20f0bd82e07bb4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bafed46a-19b2-4b02-9639-9ee993c47a10", "node_type": "1", "metadata": {}, "hash": "1c72a880bc404f2724a4e83a2118591fbae761d0802b59fea943537f2791cf1a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15754, "end_char_idx": 16957, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
869
    605 / 869
    606 / 869
    607 / 869
    608 / 869
    609 / 869
    610 / 869
    611 / 869
    612 / 869
    613 / 869
    614 / 869
    615 / 869
    616 / 869
    617 / 869
    618 / 869
    619 / 869
    620 / 869
    621 / 869
    622 / 869
    623 / 869
    624 / 869
    625 / 869
    626 / 869
    627 / 869
    628 / 869
    629 / 869
    630 / 869
    631 / 869
    632 / 869
    633 / 869
    634 / 869
    635 / 869
    636 / 869
    637 / 869
    638 / 869
    639 / 869
    640 / 869
    641 / 869
    642 / 869
    643 / 869
    644 / 869
    645 / 869
    646 / 869
    647 / 869
    648 / 869
    649 / 869
    650 / 869
    651 / 869
    652 / 869
    653 / 869
    654 / 869
    655 / 869
    656 / 869
    657 / 869
    658 / 869
    659 / 869
    660 / 869
    661 / 869
    662 / 869
    663 / 869
    664 / 869
    665 / 869
    666 / 869
    667 / 869
    668 / 869
    669 / 869
    670 / 869
    671 / 869
    672 / 869
    673 / 869
    674 / 869
    675 / 869
    676 / 869
    677 / 869
    678 / 869
    679 / 869
    680 / 869
    681 / 869
    682 / 869
    683 / 869
    684 / 869
    685 / 869
    686 / 869
    687 / 869
    688 / 869
    689 / 869
    690 /


key: _node_content
string_value:
{"id_": "bafed46a-19b2-4b02-9639-9ee993c47a10", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b1bc8604-0550-4df9-9ff7-ea04dc35f997", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6e51f4774e700394d2cea9cb43f56cb482fc96663644bc5be34053375887df0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ec096c78-10c0-4b77-87d9-534a9cb9f0e0", "node_type": "1", "metadata": {}, "hash": "a0b1434975fec54653bbbdafec9c1de685047f9e4f3b0bfc9f41c32f83c5fbef", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16336, "end_char_idx": 17535, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
646 / 869
    647 / 869
    648 / 869
    649 / 869
    650 / 869
    651 / 869
    652 / 869
    653 / 869
    654 / 869
    655 / 869
    656 / 869
    657 / 869
    658 / 869
    659 / 869
    660 / 869
    661 / 869
    662 / 869
    663 / 869
    664 / 869
    665 / 869
    666 / 869
    667 / 869
    668 / 869
    669 / 869
    670 / 869
    671 / 869
    672 / 869
    673 / 869
    674 / 869
    675 / 869
    676 / 869
    677 / 869
    678 / 869
    679 / 869
    680 / 869
    681 / 869
    682 / 869
    683 / 869
    684 / 869
    685 / 869
    686 / 869
    687 / 869
    688 / 869
    689 / 869
    690 / 869
    691 / 869
    692 / 869
    693 / 869
    694 / 869
    695 / 869
    696 / 869
    697 / 869
    698 / 869
    699 / 869
    700 / 869
    701 / 869
    702 / 869
    703 / 869
    704 / 869
    705 / 869
    706 / 869
    707 / 869
    708 / 869
    709 / 869
    710 / 869
    711 / 869
    712 / 869
    713 / 869
    714 / 869
    715 / 869
    716 / 869
    717 / 869
    718 / 869
    719 / 869
    720 / 869
    721 / 869
    722 / 869
    723 / 869
    724 / 869
    725 / 869
    726 / 869
    727 / 869
    728 / 869
    729 / 869
    730 / 869
    731 / 869


key: _node_content
string_value:
{"id_": "ec096c78-10c0-4b77-87d9-534a9cb9f0e0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "bafed46a-19b2-4b02-9639-9ee993c47a10", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "8f571feca8555788d5af0fc26774248a9c6e276e0844aa905b436316311b8cf7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6312986d-794c-46b6-8179-496a3722444c", "node_type": "1", "metadata": {}, "hash": "36e8e598e303b92e9180b4792018d84110ba0d7e356f5681466eddbadf0028eb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16924, "end_char_idx": 18123, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
688 / 869
    689 / 869
    690 / 869
    691 / 869
    692 / 869
    693 / 869
    694 / 869
    695 / 869
    696 / 869
    697 / 869
    698 / 869
    699 / 869
    700 / 869
    701 / 869
    702 / 869
    703 / 869
    704 / 869
    705 / 869
    706 / 869
    707 / 869
    708 / 869
    709 / 869
    710 / 869
    711 / 869
    712 / 869
    713 / 869
    714 / 869
    715 / 869
    716 / 869
    717 / 869
    718 / 869
    719 / 869
    720 / 869
    721 / 869
    722 / 869
    723 / 869
    724 / 869
    725 / 869
    726 / 869
    727 / 869
    728 / 869
    729 / 869
    730 / 869
    731 / 869
    732 / 869
    733 / 869
    734 / 869
    735 / 869
    736 / 869
    737 / 869
    738 / 869
    739 / 869
    740 / 869
    741 / 869
    742 / 869
    743 / 869
    744 / 869
    745 / 869
    746 / 869
    747 / 869
    748 / 869
    749 / 869
    750 / 869
    751 / 869
    752 / 869
    753 / 869
    754 / 869
    755 / 869
    756 / 869
    757 / 869
    758 / 869
    759 / 869
    760 / 869
    761 / 869
    762 / 869
    763 / 869
    764 / 869
    765 / 869
    766 / 869
    767 / 869
    768 / 869
    769 / 869
    770 / 869
    771 / 869
    772 / 869
    773 / 869


key: _node_content
string_value:
{"id_": "6312986d-794c-46b6-8179-496a3722444c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ec096c78-10c0-4b77-87d9-534a9cb9f0e0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "fe5cd2ed3e4e0cbab5075b72db5ffbd2af241f285bce689fbc5dada74ea72a6c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d97b3984-336e-4c28-9cce-e78c028bfca6", "node_type": "1", "metadata": {}, "hash": "350db7f546bcaa37594aa96547962839bd4713310c25fcac605bd24297ada823", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 17502, "end_char_idx": 18705, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
/ 869
    730 / 869
    731 / 869
    732 / 869
    733 / 869
    734 / 869
    735 / 869
    736 / 869
    737 / 869
    738 / 869
    739 / 869
    740 / 869
    741 / 869
    742 / 869
    743 / 869
    744 / 869
    745 / 869
    746 / 869
    747 / 869
    748 / 869
    749 / 869
    750 / 869
    751 / 869
    752 / 869
    753 / 869
    754 / 869
    755 / 869
    756 / 869
    757 / 869
    758 / 869
    759 / 869
    760 / 869
    761 / 869
    762 / 869
    763 / 869
    764 / 869
    765 / 869
    766 / 869
    767 / 869
    768 / 869
    769 / 869
    770 / 869
    771 / 869
    772 / 869
    773 / 869
    774 / 869
    775 / 869
    776 / 869
    777 / 869
    778 / 869
    779 / 869
    780 / 869
    781 / 869
    782 / 869
    783 / 869
    784 / 869
    785 / 869
    786 / 869
    787 / 869
    788 / 869
    789 / 869
    790 / 869
    791 / 869
    792 / 869
    793 / 869
    794 / 869
    795 / 869
    796 / 869
    797 / 869
    798 / 869
    799 / 869
    800 / 869
    801 / 869
    802 / 869
    803 / 869
    804 / 869
    805 / 869
    806 / 869
    807 / 869
    808 / 869
    809 / 869
    810 / 869
    811 / 869
    812 / 869
    813 / 869
    814 / 869
    815


key: _node_content
string_value:
{"id_": "d97b3984-336e-4c28-9cce-e78c028bfca6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6312986d-794c-46b6-8179-496a3722444c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f49a48e326d6c8aafe1018ae912dcbe68e36389409578a84e0113f4ce443816f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b0dbec75-e3f0-4eba-aa4e-06832084a2a5", "node_type": "1", "metadata": {}, "hash": "dcfe5736b44210ff7dd9bef0d431aed242da6caa4dc6610586e8d0a185b92c07", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 18086, "end_char_idx": 19285, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
771 / 869
    772 / 869
    773 / 869
    774 / 869
    775 / 869
    776 / 869
    777 / 869
    778 / 869
    779 / 869
    780 / 869
    781 / 869
    782 / 869
    783 / 869
    784 / 869
    785 / 869
    786 / 869
    787 / 869
    788 / 869
    789 / 869
    790 / 869
    791 / 869
    792 / 869
    793 / 869
    794 / 869
    795 / 869
    796 / 869
    797 / 869
    798 / 869
    799 / 869
    800 / 869
    801 / 869
    802 / 869
    803 / 869
    804 / 869
    805 / 869
    806 / 869
    807 / 869
    808 / 869
    809 / 869
    810 / 869
    811 / 869
    812 / 869
    813 / 869
    814 / 869
    815 / 869
    816 / 869
    817 / 869
    818 / 869
    819 / 869
    820 / 869
    821 / 869
    822 / 869
    823 / 869
    824 / 869
    825 / 869
    826 / 869
    827 / 869
    828 / 869
    829 / 869
    830 / 869
    831 / 869
    832 / 869
    833 / 869
    834 / 869
    835 / 869
    836 / 869
    837 / 869
    838 / 869
    839 / 869
    840 / 869
    841 / 869
    842 / 869
    843 / 869
    844 / 869
    845 / 869
    846 / 869
    847 / 869
    848 / 869
    849 / 869
    850 / 869
    851 / 869
    852 / 869
    853 / 869
    854 / 869
    855 / 869
    856 / 869


key: _node_content
string_value:
{"id_": "b0dbec75-e3f0-4eba-aa4e-06832084a2a5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ac4c727a-ede5-4d37-8af6-38661f0578e4", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0a1e8673e7bb6f84df1ef3c7408e95992d51bd540e2432875166d4aa93603de1", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d97b3984-336e-4c28-9cce-e78c028bfca6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/book_translation/translate_latex_book.ipynb", "title": "Translate a book written in LaTeX from Slovenian into English", "tokens": 8114, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "b132c767b684bee1a079a2b7133925aa6f11175f43495d3ffedf176e471cd396", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 18666, "end_char_idx": 19453, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
869
    813 / 869
    814 / 869
    815 / 869
    816 / 869
    817 / 869
    818 / 869
    819 / 869
    820 / 869
    821 / 869
    822 / 869
    823 / 869
    824 / 869
    825 / 869
    826 / 869
    827 / 869
    828 / 869
    829 / 869
    830 / 869
    831 / 869
    832 / 869
    833 / 869
    834 / 869
    835 / 869
    836 / 869
    837 / 869
    838 / 869
    839 / 869
    840 / 869
    841 / 869
    842 / 869
    843 / 869
    844 / 869
    845 / 869
    846 / 869
    847 / 869
    848 / 869
    849 / 869
    850 / 869
    851 / 869
    852 / 869
    853 / 869
    854 / 869
    855 / 869
    856 / 869
    857 / 869
    858 / 869
    859 / 869
    860 / 869
    861 / 869
    862 / 869
    863 / 869
    864 / 869
    865 / 869
    866 / 869
    867 / 869
    868 / 869


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb


key: _node_content
string_value:
{"id_": "2d5c0413-5472-4c5b-bb7b-149624d6b36c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "350bbd82-666e-419c-9d12-b3b16d36460d", "node_type": "1", "metadata": {}, "hash": "449b3241257ca1cb72749d8fed1cfd30ec0751c3a75e96bf390c9bc867ecbb9d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2466, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<span style="color:orange; font-weight:bold">Note: To answer questions based on text documents, we recommend the procedure in <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering using Embeddings</a>. Some of the code below may rely on <a href="https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints">deprecated API endpoints</a>.</span>

# 3. Train a fine-tuning model specialized for Q&A
This notebook will utilize the dataset of context, question and answer pairs to additionally create adversarial questions and context pairs, where the question was not generated on that context. In those cases the model will be prompted to answer "No sufficient context for answering the question". We will also train a discriminator model, which predicts whether the question can be answered based on the context or not.

We will add hard adversarial examples as well, which will be based either on semantically similar sections, or neighbouring sections, originating from the same article.


```python
import openai
import pandas as pd
df = pd.read_csv('olympics-data/olympics_qa.csv')
olympics_search_fileid = "file-c3shd8wqF3vSCKaukW4Jr1TT"
df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>heading</th>
      <th>content</th>
      <th>tokens</th>
      <th>context</th>
      <th>questions</th>
      <th>answers</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>2020 Summer Olympics</td>
      <td>Summary</td>
      <td>The 2020 Summer Olympics (Japanese: 2020年夏季オリン...</td>
      <td>713</td>
      <td>2020 Summer Olympics\nSummary\n\nThe 2020 Summ...</td>
      <td>1. What is the 2020 Summer Olympics?\n2. When ...</td>
      <td>1. The 2020 Summer Olympics is an internationa...</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2020 Summer Olympics</td>
      <td>Host city selection</td>
      <td>The International Olympic Committee (IOC) vote...</td>
      <td>126</td>
      <td>2020 Summer Olympics\nHost city selection\n\nT...</td>
      <td>1. \n2. \n3. \n4.</td>
      <td>1.


key: _node_content
string_value:
{"id_": "350bbd82-666e-419c-9d12-b3b16d36460d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2d5c0413-5472-4c5b-bb7b-149624d6b36c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d1ce3341ada6351811320c9b3c34779b231b2f0cfed54786fd7368f9dfa9981b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0a0d99ee-5069-47cb-8827-322e8e5a989b", "node_type": "1", "metadata": {}, "hash": "f12c67abb246a9f846d950c368a933c933fe9360d9809f903f302b6f620653cd", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1276, "end_char_idx": 3670, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>heading</th>
      <th>content</th>
      <th>tokens</th>
      <th>context</th>
      <th>questions</th>
      <th>answers</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>2020 Summer Olympics</td>
      <td>Summary</td>
      <td>The 2020 Summer Olympics (Japanese: 2020年夏季オリン...</td>
      <td>713</td>
      <td>2020 Summer Olympics\nSummary\n\nThe 2020 Summ...</td>
      <td>1. What is the 2020 Summer Olympics?\n2. When ...</td>
      <td>1. The 2020 Summer Olympics is an internationa...</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2020 Summer Olympics</td>
      <td>Host city selection</td>
      <td>The International Olympic Committee (IOC) vote...</td>
      <td>126</td>
      <td>2020 Summer Olympics\nHost city selection\n\nT...</td>
      <td>1. \n2. \n3. \n4.</td>
      <td>1. What is the International Olympic Committee...</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2020 Summer Olympics</td>
      <td>Impact of the COVID-19 pandemic</td>
      <td>In January 2020, concerns were raised about th...</td>
      <td>369</td>
      <td>2020 Summer Olympics\nImpact of the COVID-19 p...</td>
      <td>1. What was the COVID-19 pandemic?\n2. How did...</td>
      <td>1. The COVID-19 pandemic was a pandemic that o...</td>
    </tr>
    <tr>
      <th>3</th>
      <td>2020 Summer Olympics</td>
      <td>Qualifying event cancellation and postponement</td>
      <td>Concerns about the pandemic began to affect qu...</td>
      <td>298</td>
      <td>2020 Summer Olympics\nQualifying event cancell...</td>
      <td>1. What was the original location of the Asia ...</td>
      <td>1. The original location of the Asia &amp; Oceania...</td>
    </tr>
    <tr>
      <th>4</th>
      <td>2020 Summer Olympics</td>
      <td>Effect on doping tests</td>
      <td>Mandatory doping tests were being severely res...</td>
      <td>163</td>
      <td>2020 Summer Olympics\nEffect on doping tests\n...</td>
      <td>1. What was the COVID-19 pandemic?\n2. What di...</td>
      <td>1.


key: _node_content
string_value:
{"id_": "0a0d99ee-5069-47cb-8827-322e8e5a989b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "350bbd82-666e-419c-9d12-b3b16d36460d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "fa18bbfccc43c200190693145fa25609d4b94ae459a4ab8b083903b7298aca91", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c47ad0c9-7f48-4552-bf84-b48013197300", "node_type": "1", "metadata": {}, "hash": "85546658aff985b4d27e4eee28ab43cccb8c549be21360871bda44c87310aeed", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2434, "end_char_idx": 5075, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
\n2. \n3. \n4.</td>
      <td>1. What is the International Olympic Committee...</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2020 Summer Olympics</td>
      <td>Impact of the COVID-19 pandemic</td>
      <td>In January 2020, concerns were raised about th...</td>
      <td>369</td>
      <td>2020 Summer Olympics\nImpact of the COVID-19 p...</td>
      <td>1. What was the COVID-19 pandemic?\n2. How did...</td>
      <td>1. The COVID-19 pandemic was a pandemic that o...</td>
    </tr>
    <tr>
      <th>3</th>
      <td>2020 Summer Olympics</td>
      <td>Qualifying event cancellation and postponement</td>
      <td>Concerns about the pandemic began to affect qu...</td>
      <td>298</td>
      <td>2020 Summer Olympics\nQualifying event cancell...</td>
      <td>1. What was the original location of the Asia ...</td>
      <td>1. The original location of the Asia &amp; Oceania...</td>
    </tr>
    <tr>
      <th>4</th>
      <td>2020 Summer Olympics</td>
      <td>Effect on doping tests</td>
      <td>Mandatory doping tests were being severely res...</td>
      <td>163</td>
      <td>2020 Summer Olympics\nEffect on doping tests\n...</td>
      <td>1. What was the COVID-19 pandemic?\n2. What di...</td>
      <td>1. The COVID-19 pandemic was a pandemic that o...</td>
    </tr>
  </tbody>
</table>
</div>



Split the sections into a training and testing set


```python
from sklearn.model_selection import train_test_split
train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)
len(train_df), len(test_df)
```




    (3014, 754)



we check that the separator we intend to use isn't present within the contexts


```python
df.context.str.contains('->').sum()
```




    0



## 3.1 Create the fine-tuning datasets for Q&A and discriminator models
The fine-tuning dataset is created in the following way. For every corresponding question, answer and context pair we create:
- Positive example: correct question, answer, context pair
- Negative examples:
  - random negative example, where the random context is paired with the question 
  - two hard negative examples
    - one originating from the same wikipedia article
    - another, which is most similar to the correct context

This process is noisy, as sometimes the question might be answerable given a different context, but on average we hope this won't affect the performance too much.

We apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.


key: _node_content
string_value:
{"id_": "c47ad0c9-7f48-4552-bf84-b48013197300", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0a0d99ee-5069-47cb-8827-322e8e5a989b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d4e009f49cbefb5395ffbcbf880db7fef123375970d873385f046aa9359e2dc2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "34b30c3e-97d7-40df-a8dd-adc54e8ced38", "node_type": "1", "metadata": {}, "hash": "733bf2b599de79aef180c1639506e25efc36cd90590dc885243e4fe29e061b32", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3606, "end_char_idx": 6839, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
What was the COVID-19 pandemic?\n2. What di...</td>
      <td>1. The COVID-19 pandemic was a pandemic that o...</td>
    </tr>
  </tbody>
</table>
</div>



Split the sections into a training and testing set


```python
from sklearn.model_selection import train_test_split
train_df, test_df = train_test_split(df, test_size=0.2, random_state=42)
len(train_df), len(test_df)
```




    (3014, 754)



we check that the separator we intend to use isn't present within the contexts


```python
df.context.str.contains('->').sum()
```




    0



## 3.1 Create the fine-tuning datasets for Q&A and discriminator models
The fine-tuning dataset is created in the following way. For every corresponding question, answer and context pair we create:
- Positive example: correct question, answer, context pair
- Negative examples:
  - random negative example, where the random context is paired with the question 
  - two hard negative examples
    - one originating from the same wikipedia article
    - another, which is most similar to the correct context

This process is noisy, as sometimes the question might be answerable given a different context, but on average we hope this won't affect the performance too much.

We apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.


```python
import random

def get_random_similar_contexts(question, context, file_id=olympics_search_fileid, search_model='ada', max_rerank=10):
    """
    Find similar contexts to the given context using the search file
    """
    try:
        # TODO: openai.Engine(search_model) is deprecated
        results = openai.Engine(search_model).search(
            search_model=search_model, 
            query=question, 
            max_rerank=max_rerank,
            file=file_id
        )
        candidates = []
        for result in results['data'][:3]:
            if result['text'] == context:
                continue
            candidates.append(result['text'])
        random_candidate = random.choice(candidates)
        return random_candidate
    except Exception as e:
        print(e)
        return ""

def create_fine_tuning_dataset(df, discriminator=False, n_negative=1, add_related=False):
    """
    Create a dataset for fine tuning the OpenAI model; either for a discriminator model, 
    or a model specializing in Q&A, where it says if no relevant context is found.

    Parameters
    ----------
    df: pd.DataFrame
        The dataframe containing the question, answer and context pairs
    discriminator: bool
        Whether to create a dataset for the discriminator
    n_negative: int
        The number of random negative samples to add (using a random context)
    add_related: bool
        Whether to add the related contexts to the correct context. These are hard negative examples

    Returns
    -------
    pd.DataFrame
        The dataframe containing the prompts and completions, ready for fine-tuning
    """
    rows = []
    for i, row in df.iterrows():
        for q, a in zip(("1." + row.questions).split('\n'), ("1."


key: _node_content
string_value:
{"id_": "34b30c3e-97d7-40df-a8dd-adc54e8ced38", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c47ad0c9-7f48-4552-bf84-b48013197300", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f64af391e3d219d48b9c0d4187474c00ad6ac7c4f830468b5896d5fec20d2b0d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7ab5e820-9360-4c2a-b61d-69e86f6be596", "node_type": "1", "metadata": {}, "hash": "7cd47454946ab6139afae71ab201585ccc7d5c8834aa578c5fc36a089c5db99f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5078, "end_char_idx": 7275, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import random

def get_random_similar_contexts(question, context, file_id=olympics_search_fileid, search_model='ada', max_rerank=10):
    """
    Find similar contexts to the given context using the search file
    """
    try:
        # TODO: openai.Engine(search_model) is deprecated
        results = openai.Engine(search_model).search(
            search_model=search_model, 
            query=question, 
            max_rerank=max_rerank,
            file=file_id
        )
        candidates = []
        for result in results['data'][:3]:
            if result['text'] == context:
                continue
            candidates.append(result['text'])
        random_candidate = random.choice(candidates)
        return random_candidate
    except Exception as e:
        print(e)
        return ""

def create_fine_tuning_dataset(df, discriminator=False, n_negative=1, add_related=False):
    """
    Create a dataset for fine tuning the OpenAI model; either for a discriminator model, 
    or a model specializing in Q&A, where it says if no relevant context is found.

    Parameters
    ----------
    df: pd.DataFrame
        The dataframe containing the question, answer and context pairs
    discriminator: bool
        Whether to create a dataset for the discriminator
    n_negative: int
        The number of random negative samples to add (using a random context)
    add_related: bool
        Whether to add the related contexts to the correct context. These are hard negative examples

    Returns
    -------
    pd.DataFrame
        The dataframe containing the prompts and completions, ready for fine-tuning
    """
    rows = []
    for i, row in df.iterrows():
        for q, a in zip(("1." + row.questions).split('\n'), ("1." + row.answers).split('\n')):
            if len(q) >10 and len(a) >10:
                if discriminator:
                    rows.append({"prompt":f"{row.context}\nQuestion: {q[2:].strip()}\n Related:", "completion":f" yes"})
                else:
                    rows.append({"prompt":f"{row.context}\nQuestion: {q[2:].strip()}\nAnswer:", "completion":f" {a[2:].strip()}"})

    for i, row in df.iterrows():
        for q in ("1."


key: _node_content
string_value:
{"id_": "7ab5e820-9360-4c2a-b61d-69e86f6be596", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "34b30c3e-97d7-40df-a8dd-adc54e8ced38", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "0e7d6e4cd48b2b65afb3b048a7eb02e82e76c4ce9bcc3de23db8cc7f488f17c5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c5040ed6-6561-4936-9b8b-594c214570af", "node_type": "1", "metadata": {}, "hash": "4952c8c96d01674ece45e094eebc13b338fd2ce070d706a4f9c1675cac1e3b7b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6171, "end_char_idx": 9452, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Parameters
    ----------
    df: pd.DataFrame
        The dataframe containing the question, answer and context pairs
    discriminator: bool
        Whether to create a dataset for the discriminator
    n_negative: int
        The number of random negative samples to add (using a random context)
    add_related: bool
        Whether to add the related contexts to the correct context. These are hard negative examples

    Returns
    -------
    pd.DataFrame
        The dataframe containing the prompts and completions, ready for fine-tuning
    """
    rows = []
    for i, row in df.iterrows():
        for q, a in zip(("1." + row.questions).split('\n'), ("1." + row.answers).split('\n')):
            if len(q) >10 and len(a) >10:
                if discriminator:
                    rows.append({"prompt":f"{row.context}\nQuestion: {q[2:].strip()}\n Related:", "completion":f" yes"})
                else:
                    rows.append({"prompt":f"{row.context}\nQuestion: {q[2:].strip()}\nAnswer:", "completion":f" {a[2:].strip()}"})

    for i, row in df.iterrows():
        for q in ("1." + row.questions).split('\n'):
            if len(q) >10:
                for j in range(n_negative + (2 if add_related else 0)):
                    random_context = ""
                    if j == 0 and add_related:
                        # add the related contexts based on originating from the same wikipedia page
                        subset = df[(df.title == row.title) & (df.context != row.context)]
                        
                        if len(subset) < 1:
                            continue
                        random_context = subset.sample(1).iloc[0].context
                    if j == 1 and add_related:
                        # add the related contexts based on the most similar contexts according to the search
                        random_context = get_random_similar_contexts(q[2:].strip(), row.context, search_model='ada', max_rerank=10)
                    else:
                        while True:
                            # add random context, which isn't the correct context
                            random_context = df.sample(1).iloc[0].context
                            if random_context != row.context:
                                break
                    if discriminator:
                        rows.append({"prompt":f"{random_context}\nQuestion: {q[2:].strip()}\n Related:", "completion":f" no"})
                    else:
                        rows.append({"prompt":f"{random_context}\nQuestion: {q[2:].strip()}\nAnswer:", "completion":f" No appropriate context found to answer the question."})

    return pd.DataFrame(rows) 
```

We apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.


```python
for name, is_disc in [('discriminator', True), ('qa', False)]:
    for train_test, dt in [('train', train_df), ('test', test_df)]:
        ft = create_fine_tuning_dataset(dt, discriminator=is_disc, n_negative=1, add_related=True)
        ft.to_json(f'{name}_{train_test}.jsonl', orient='records', lines=True)
```


key: _node_content
string_value:
{"id_": "c5040ed6-6561-4936-9b8b-594c214570af", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7ab5e820-9360-4c2a-b61d-69e86f6be596", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "2824c393eb81a87f87cd12801cb9e646b8f12657d1ffb3d5e3aa3873af1b7418", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1364e36d-9503-447b-a938-e47969cf9147", "node_type": "1", "metadata": {}, "hash": "480a487e2559ab1c532fa75f9b253037d81cee8e2b0cdc80cebb6e7fc7199876", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8842, "end_char_idx": 11388, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
return pd.DataFrame(rows) 
```

We apply the same process of dataset creation for both the discriminator, and the Q&A answering model. We apply the process separately for the training and testing set, to ensure that the examples from the training set don't feature within the test set.


```python
for name, is_disc in [('discriminator', True), ('qa', False)]:
    for train_test, dt in [('train', train_df), ('test', test_df)]:
        ft = create_fine_tuning_dataset(dt, discriminator=is_disc, n_negative=1, add_related=True)
        ft.to_json(f'{name}_{train_test}.jsonl', orient='records', lines=True)
```




    



We formatted the data according to the recommendations from the fine-tuning tool, which is available using
> openai tools fine_tunes.prepare_data -f qa_train.jsonl

We highly recommend that you use this tool, which suggests improvements in your data formatting for fine-tuning.


## 3.2 Submit the datasets for fine-tuning


```python
!openai api fine_tunes.create -t "olympics-data/discriminator_train.jsonl" -v "olympics-data/discriminator_test.jsonl" --batch_size 16  --compute_classification_metrics --classification_positive_class " yes" --model ada
```




    




```python
!openai api fine_tunes.create -t "olympics-data/qa_train.jsonl" -v "olympics-data/qa_test.jsonl" --batch_size 16
```




    



## 3.3 Using the fine-tuned models

We will now use the fine-tuned discriminator and the fine-tuned Q&A model. By requesting logprobs, we can see how certain the discriminator is in a `yes` vs `no` answer.


```python
ft_discriminator = "curie:ft-openai-internal-2021-08-23-23-58-57"
ft_qa = "curie:ft-openai-internal-2021-08-23-17-54-10"

def apply_ft_discriminator(context, question, discriminator_model):
    """
    Apply the fine tuned discriminator to a question, to assess whether it can be answered from the context.
    """
    prompt = f"{context}\nQuestion: {question}\n Related:"
    result = openai.chat.completions.create(model=discriminator_model, prompt=prompt, max_tokens=1, temperature=0, top_p=1, n=1, logprobs=2)
    return result['choices'][0]['logprobs']['top_logprobs']

apply_ft_discriminator('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', 
                        'What was the first human-made object in space?', ft_discriminator)
```




    [<OpenAIObject at 0x7fe812e602b0> JSON: {
       " no": -10.819577,
       " yes": -2.045765e-05
     }]



We can see that the model can generalize well to different contexts and questions.


key: _node_content
string_value:
{"id_": "1364e36d-9503-447b-a938-e47969cf9147", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c5040ed6-6561-4936-9b8b-594c214570af", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "bb266521f3e9c76b049097d1880ecc80308124e96d0c5cd75fc2e0b3ee03bda2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4472939a-de34-412e-a117-256888e921a0", "node_type": "1", "metadata": {}, "hash": "27ff64ed005db3e16c69aebcedb676879b47791f883923d5c60482304732d79e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10037, "end_char_idx": 12758, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
!openai api fine_tunes.create -t "olympics-data/qa_train.jsonl" -v "olympics-data/qa_test.jsonl" --batch_size 16
```




    



## 3.3 Using the fine-tuned models

We will now use the fine-tuned discriminator and the fine-tuned Q&A model. By requesting logprobs, we can see how certain the discriminator is in a `yes` vs `no` answer.


```python
ft_discriminator = "curie:ft-openai-internal-2021-08-23-23-58-57"
ft_qa = "curie:ft-openai-internal-2021-08-23-17-54-10"

def apply_ft_discriminator(context, question, discriminator_model):
    """
    Apply the fine tuned discriminator to a question, to assess whether it can be answered from the context.
    """
    prompt = f"{context}\nQuestion: {question}\n Related:"
    result = openai.chat.completions.create(model=discriminator_model, prompt=prompt, max_tokens=1, temperature=0, top_p=1, n=1, logprobs=2)
    return result['choices'][0]['logprobs']['top_logprobs']

apply_ft_discriminator('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', 
                        'What was the first human-made object in space?', ft_discriminator)
```




    [<OpenAIObject at 0x7fe812e602b0> JSON: {
       " no": -10.819577,
       " yes": -2.045765e-05
     }]



We can see that the model can generalize well to different contexts and questions. 


```python
def apply_ft_qa_answer(context, question, answering_model):
    """
    Apply the fine tuned discriminator to a question
    """
    prompt = f"{context}\nQuestion: {question}\nAnswer:"
    result = openai.chat.completions.create(model=answering_model, prompt=prompt, max_tokens=30, temperature=0, top_p=1, n=1, stop=['.','\n'])
    return result['choices'][0]['text']

apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', 
                    'What was the first human-made object in space?', ft_qa)

```




    ' The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957'



We can see that the model can answer the question, when the context is appropriate.


```python
apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',
                    'What is impressive about the Soviet Union?', ft_qa)
```




    ' The Soviet Union was the first country to successfully launch a satellite into space'




```python
apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',
                    'How many cars were produced in the Soviet Union in 1970?', ft_qa)
```




    ' No appropriate context found to answer the question'


key: _node_content
string_value:
{"id_": "4472939a-de34-412e-a117-256888e921a0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1364e36d-9503-447b-a938-e47969cf9147", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "86cba393c0e54b27fcf9e8d6542829db024d4038284898bd3ba283621ca2c53e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c0d0d7eb-4755-4877-94ea-5eaa180a5037", "node_type": "1", "metadata": {}, "hash": "dd0e2870aa9a41a060aa057792de758d51ceea743c78493f1bdf148ccfa7b3ba", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11198, "end_char_idx": 14490, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
[<OpenAIObject at 0x7fe812e602b0> JSON: {
       " no": -10.819577,
       " yes": -2.045765e-05
     }]



We can see that the model can generalize well to different contexts and questions. 


```python
def apply_ft_qa_answer(context, question, answering_model):
    """
    Apply the fine tuned discriminator to a question
    """
    prompt = f"{context}\nQuestion: {question}\nAnswer:"
    result = openai.chat.completions.create(model=answering_model, prompt=prompt, max_tokens=30, temperature=0, top_p=1, n=1, stop=['.','\n'])
    return result['choices'][0]['text']

apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.', 
                    'What was the first human-made object in space?', ft_qa)

```




    ' The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957'



We can see that the model can answer the question, when the context is appropriate.


```python
apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',
                    'What is impressive about the Soviet Union?', ft_qa)
```




    ' The Soviet Union was the first country to successfully launch a satellite into space'




```python
apply_ft_qa_answer('The first human-made object in space was the Soviet Union satellite Sputnik 1 on 4 October 1957.',
                    'How many cars were produced in the Soviet Union in 1970?', ft_qa)
```




    ' No appropriate context found to answer the question'



We can see that the model knows when to answer the question, and when to say that insufficient context is present to answer the question.

We can also combine a discriminator and a base model, or a fine-tuned Q&A model. Discriminator can essentially serve as a decision whether the question can be answered given the context or not.


```python
def answer_question_conditionally(answering_model, discriminator_model, context, question, discriminator_logprob_yes_modifier=0):
    logprobs = apply_ft_discriminator(context, question, discriminator_model)
    yes_logprob = logprobs[' yes'] if ' yes' in logprobs else -100
    no_logprob = logprobs[' no'] if ' no' in logprobs else -100
    if yes_logprob + discriminator_logprob_yes_modifier < no_logprob:
        return " No appropriate context found to answer the question based on the discriminator."
    return apply_ft_qa_answer(context, question, answering_model)
answer_question_conditionally(ft_qa, ft_discriminator, 
                                "Crowdless games are a rare although not unheard-of occurrence in sports. \
                                 When they do occur, it is usually the result of events beyond the control \
                                 of the teams or fans, such as weather-related concerns, public health concerns, \
                                 or wider civil disturbances unrelated to the game. For instance, \
                                 the COVID-19 pandemic caused many sports leagues around the world \
                                 to be played behind closed doors.",
                                "Could weather cause a sport event to have no crowd?")
```




    ' Weather could cause a sport event to have no crowd'


key: _node_content
string_value:
{"id_": "c0d0d7eb-4755-4877-94ea-5eaa180a5037", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "3b6d8703-b175-4cc1-9d68-76a4adde5446", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f254b8aa71e3e1906f07ce37ffb70b3c250910fc2ccc3f31ca144d7e478357e4", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4472939a-de34-412e-a117-256888e921a0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-3-train-qa.ipynb", "title": "3. Train a fine-tuning model specialized for Q&A", "tokens": 3706, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c21064ea8f2663339fb675bcc43976902e954243f7556353a11c1e4c35487814", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12704, "end_char_idx": 15566, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
' No appropriate context found to answer the question'



We can see that the model knows when to answer the question, and when to say that insufficient context is present to answer the question.

We can also combine a discriminator and a base model, or a fine-tuned Q&A model. Discriminator can essentially serve as a decision whether the question can be answered given the context or not.


```python
def answer_question_conditionally(answering_model, discriminator_model, context, question, discriminator_logprob_yes_modifier=0):
    logprobs = apply_ft_discriminator(context, question, discriminator_model)
    yes_logprob = logprobs[' yes'] if ' yes' in logprobs else -100
    no_logprob = logprobs[' no'] if ' no' in logprobs else -100
    if yes_logprob + discriminator_logprob_yes_modifier < no_logprob:
        return " No appropriate context found to answer the question based on the discriminator."
    return apply_ft_qa_answer(context, question, answering_model)
answer_question_conditionally(ft_qa, ft_discriminator, 
                                "Crowdless games are a rare although not unheard-of occurrence in sports. \
                                 When they do occur, it is usually the result of events beyond the control \
                                 of the teams or fans, such as weather-related concerns, public health concerns, \
                                 or wider civil disturbances unrelated to the game. For instance, \
                                 the COVID-19 pandemic caused many sports leagues around the world \
                                 to be played behind closed doors.",
                                "Could weather cause a sport event to have no crowd?")
```




    ' Weather could cause a sport event to have no crowd'



The above function illustrates how to potentially combine a discriminator and a fine-tuned Q&A model. This gives a more fine-grained control over how certain we want the model to be before it answers the question.

We'll now take a look on how answers endpoint works - combining search to retrieve the relevant context from a knowledge base, and then using the fine-tuned Q&A model to answer the question.

## 3.4 Answering the question based on a knowledge base
Finally we can use a logic similar to the [/answers](https://beta.openai.com/docs/api-reference/answers) endpoint, where we first search for the relevant context, and then ask a Q&A model to answer the question given that context. If you'd like to see the implementation details, check out the [`answers_with_ft.py`](answers_with_ft.py) file.


```python
from answers_with_ft import answer_question
answer_question(olympics_search_fileid, ft_qa, "Which country won the Women's football tournament at the 2020 Olympic games?")
```




    " Canada won the Women's football tournament at the 2020 Olympic games"


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb


key: _node_content
string_value:
{"id_": "8c5058a7-b386-4c4a-af49-f35e73cb8d93", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "378c6165-3248-437f-bbb6-bcf14e8f390e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3a8a6133f9a91f6c5d7e64f055a62a395bae57425cd2b65daa6f49d3ba186d16", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "636aade6-d9b7-460f-99d7-8f98098b4128", "node_type": "1", "metadata": {}, "hash": "abf35b6ef3cb253f1c316c5be58c6c77bc7ed80f0ffa7eec342ad985636ebf89", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2537, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<span style="color:orange; font-weight:bold">Note: To answer questions based on text documents, we recommend the procedure in <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering using Embeddings</a>. Some of the code below may rely on <a href="https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints">deprecated API endpoints</a>.</span>

# 1. Collect Wikipedia data about Olympic Games 2020

The idea of this project is to create a question answering model, based on a few paragraphs of provided text. Base GPT-3 models do a good job at answering questions when the answer is contained within the paragraph, however if the answer isn't contained, the base models tend to try their best to answer anyway, often leading to confabulated answers. 

To create a model which answers questions only if there is sufficient context for doing so, we first create a dataset of questions and answers based on paragraphs of text. In order to train the model to answer only when the answer is present, we also add adversarial examples, where the question doesn't match the context. In those cases, we ask the model to output "No sufficient context for answering the question". 

We will perform this task in three notebooks:
1. The first (this) notebook focuses on collecting recent data, which GPT-3 didn't see during it's pre-training. We picked the topic of Olympic Games 2020 (which actually took place in the summer of 2021), and downloaded 713 unique pages. We organized the dataset by individual sections, which will serve as context for asking and answering the questions.
2. The [second notebook](olympics-2-create-qa.ipynb) will utilize Davinci-instruct to ask a few questions based on a Wikipedia section, as well as answer those questions, based on that section.
3. The [third notebook](olympics-3-train-qa.ipynb) will utilize the dataset of context, question and answer pairs to additionally create adversarial questions and context pairs, where the question was not generated on that context. In those cases the model will be prompted to answer "No sufficient context for answering the question". We will also train a discriminator model, which predicts whether the question can be answered based on the context or not.

## 1.1 Data extraction using the wikipedia API
Extracting the data will take about half an hour, and processing will likely take about as much.


```python
import pandas as pd
import wikipedia


key: _node_content
string_value:
{"id_": "636aade6-d9b7-460f-99d7-8f98098b4128", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "378c6165-3248-437f-bbb6-bcf14e8f390e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3a8a6133f9a91f6c5d7e64f055a62a395bae57425cd2b65daa6f49d3ba186d16", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8c5058a7-b386-4c4a-af49-f35e73cb8d93", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "daa464c73cf0ffe8b2542ec7a1989e1d23f88b43d6ca1b20c5da40b71ca9c5ce", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3bb714d6-d1ea-46da-bc4c-6271356e4e0a", "node_type": "1", "metadata": {}, "hash": "8a3a087c022e785b48b6f357cb462adc284d1c5d1116f1be227075e044f7641a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2491, "end_char_idx": 5383, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import pandas as pd
import wikipedia


def filter_olympic_2020_titles(titles):
    """
    Get the titles which are related to Olympic games hosted in 2020, given a list of titles
    """
    titles = [title for title in titles if '2020' in title and 'olympi' in title.lower()]
    
    return titles

def get_wiki_page(title):
    """
    Get the wikipedia page given a title
    """
    try:
        return wikipedia.page(title)
    except wikipedia.exceptions.DisambiguationError as e:
        return wikipedia.page(e.options[0])
    except wikipedia.exceptions.PageError as e:
        return None

def recursively_find_all_pages(titles, titles_so_far=set()):
    """
    Recursively find all the pages that are linked to the Wikipedia titles in the list
    """
    all_pages = []
    
    titles = list(set(titles) - titles_so_far)
    titles = filter_olympic_2020_titles(titles)
    titles_so_far.update(titles)
    for title in titles:
        page = get_wiki_page(title)
        if page is None:
            continue
        all_pages.append(page)

        new_pages = recursively_find_all_pages(page.links, titles_so_far)
        for pg in new_pages:
            if pg.title not in [p.title for p in all_pages]:
                all_pages.append(pg)
        titles_so_far.update(page.links)
    return all_pages


pages = recursively_find_all_pages(["2020 Summer Olympics"])
len(pages)
```




    909



## 1.2 Filtering the Wikipedia pages and splitting them into sections by headings
We remove sections unlikely to contain textual information, and ensure that each section is not longer than the token limit


```python

import re
from typing import Set
from transformers import GPT2TokenizerFast

import numpy as np
from nltk.tokenize import sent_tokenize

tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")

def count_tokens(text: str) -> int:
    """count the number of tokens in a string"""
    return len(tokenizer.encode(text))

def reduce_long(
    long_text: str, long_text_tokens: bool = False, max_len: int = 590
) -> str:
    """
    Reduce a long text to a maximum of `max_len` tokens by potentially cutting at a sentence end
    """
    if not long_text_tokens:
        long_text_tokens = count_tokens(long_text)
    if long_text_tokens > max_len:
        sentences = sent_tokenize(long_text.replace("\n", " "))
        ntokens = 0
        for i, sentence in enumerate(sentences):
            ntokens += 1 + count_tokens(sentence)
            if ntokens > max_len:
                return ". ".join(sentences[:i]) + "."

    return long_text

discard_categories = ['See also', 'References', 'External links', 'Further reading', "Footnotes",
    "Bibliography", "Sources", "Citations", "Literature", "Footnotes", "Notes and references",
    "Photo gallery", "Works cited", "Photos", "Gallery", "Notes", "References and sources",
    "References and notes",]


key: _node_content
string_value:
{"id_": "3bb714d6-d1ea-46da-bc4c-6271356e4e0a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "378c6165-3248-437f-bbb6-bcf14e8f390e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3a8a6133f9a91f6c5d7e64f055a62a395bae57425cd2b65daa6f49d3ba186d16", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "636aade6-d9b7-460f-99d7-8f98098b4128", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "48394ece7edf83343860ae3d4d0418240d4164a253ba6c2cc96b70fb27425bf7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9c470077-244b-4407-8fd0-440755f6f067", "node_type": "1", "metadata": {}, "hash": "56fa6a030c3d7a503e8064cea6a1b7c497fb0554dca98016efbe46ad3037f503", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3823, "end_char_idx": 7821, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
pages = recursively_find_all_pages(["2020 Summer Olympics"])
len(pages)
```




    909



## 1.2 Filtering the Wikipedia pages and splitting them into sections by headings
We remove sections unlikely to contain textual information, and ensure that each section is not longer than the token limit


```python

import re
from typing import Set
from transformers import GPT2TokenizerFast

import numpy as np
from nltk.tokenize import sent_tokenize

tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")

def count_tokens(text: str) -> int:
    """count the number of tokens in a string"""
    return len(tokenizer.encode(text))

def reduce_long(
    long_text: str, long_text_tokens: bool = False, max_len: int = 590
) -> str:
    """
    Reduce a long text to a maximum of `max_len` tokens by potentially cutting at a sentence end
    """
    if not long_text_tokens:
        long_text_tokens = count_tokens(long_text)
    if long_text_tokens > max_len:
        sentences = sent_tokenize(long_text.replace("\n", " "))
        ntokens = 0
        for i, sentence in enumerate(sentences):
            ntokens += 1 + count_tokens(sentence)
            if ntokens > max_len:
                return ". ".join(sentences[:i]) + "."

    return long_text

discard_categories = ['See also', 'References', 'External links', 'Further reading', "Footnotes",
    "Bibliography", "Sources", "Citations", "Literature", "Footnotes", "Notes and references",
    "Photo gallery", "Works cited", "Photos", "Gallery", "Notes", "References and sources",
    "References and notes",]


def extract_sections(
    wiki_text: str,
    title: str,
    max_len: int = 1500,
    discard_categories: Set[str] = discard_categories,
) -> str:
    """
    Extract the sections of a Wikipedia page, discarding the references and other low information sections
    """
    if len(wiki_text) == 0:
        return []

    # find all headings and the corresponding contents
    headings = re.findall("==+ .* ==+", wiki_text)
    for heading in headings:
        wiki_text = wiki_text.replace(heading, "==+ !! ==+")
    contents = wiki_text.split("==+ !! ==+")
    contents = [c.strip() for c in contents]
    assert len(headings) == len(contents) - 1

    cont = contents.pop(0).strip()
    outputs = [(title, "Summary", cont, count_tokens(cont)+4)]
    
    # discard the discard categories, accounting for a tree structure
    max_level = 100
    keep_group_level = max_level
    remove_group_level = max_level
    nheadings, ncontents = [], []
    for heading, content in zip(headings, contents):
        plain_heading = " ".join(heading.split(" ")[1:-1])
        num_equals = len(heading.split(" ")[0])
        if num_equals <= keep_group_level:
            keep_group_level = max_level

        if num_equals > remove_group_level:
            if (
                num_equals <= keep_group_level
            ):
                continue
        keep_group_level = max_level
        if plain_heading in discard_categories:
            remove_group_level = num_equals
            keep_group_level = max_level
            continue
        nheadings.append(heading.replace("=", "").strip())
        ncontents.append(content)
        remove_group_level = max_level

    # count the tokens of each section
    ncontent_ntokens = [
        count_tokens(c)
        + 3
        + count_tokens(" ".join(h.split(" ")[1:-1]))
        - (1 if len(c) == 0 else 0)
        for h, c in zip(nheadings, ncontents)
    ]

    # Create a tuple of (title, section_name, content, number of tokens)
    outputs += [(title, h, c, t) if t<max_len 
                else (title, h, reduce_long(c, max_len), count_tokens(reduce_long(c,max_len))) 
                    for h, c, t in zip(nheadings, ncontents, ncontent_ntokens)]
    
    return outputs

# Example page being processed into sections
bermuda_page = get_wiki_page('Bermuda at the 2020 Summer Olympics')
ber = extract_sections(bermuda_page.content, bermuda_page.title)

# Example section
ber[-1]

```


key: _node_content
string_value:
{"id_": "9c470077-244b-4407-8fd0-440755f6f067", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "378c6165-3248-437f-bbb6-bcf14e8f390e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3a8a6133f9a91f6c5d7e64f055a62a395bae57425cd2b65daa6f49d3ba186d16", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3bb714d6-d1ea-46da-bc4c-6271356e4e0a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "4d1726e73d28316c3f7bc411ddfac2bd6a705d6eaafcc1b3b525a053f587a81f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "141f71a5-7b4d-46f8-8944-24611dced5c6", "node_type": "1", "metadata": {}, "hash": "0074b278e894572f27120ec3f93bd591ddd371d9a6339b288dc192a86b320b81", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7830, "end_char_idx": 10514, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
('Bermuda at the 2020 Summer Olympics',
     'Equestrian',
     "Bermuda entered one dressage rider into the Olympic competition by finishing in the top four, outside the group selection, of the individual FEI Olympic Rankings for Groups D and E (North, Central, and South America), marking the country's recurrence to the sport after an eight-year absence. The quota was later withdrawn, following an injury of Annabelle Collins' main horse Joyero and a failure to obtain minimum eligibility requirements (MER) aboard a new horse Chuppy Checker.",
     104)



### 1.2.1 We create a dataset and filter out any sections with fewer than 40 tokens, as those are unlikely to contain enough context to ask a good question.


```python
res = []
for page in pages:
    res += extract_sections(page.content, page.title)
df = pd.DataFrame(res, columns=["title", "heading", "content", "tokens"])
df = df[df.tokens>40]
df = df.drop_duplicates(['title','heading'])
df = df.reset_index().drop('index',axis=1) # reset index
df.head()
```

    Token indices sequence length is longer than the specified maximum sequence length for this model (1060 > 1024). Running this sequence through the model will result in indexing errors





<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>heading</th>
      <th>content</th>
      <th>tokens</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>2020 Summer Olympics</td>
      <td>Summary</td>
      <td>The 2020 Summer Olympics (Japanese: 2020年夏季オリン...</td>
      <td>713</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2020 Summer Olympics</td>
      <td>Host city selection</td>
      <td>The International Olympic Committee (IOC) vote...</td>
      <td>126</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2020 Summer Olympics</td>
      <td>Impact of the COVID-19 pandemic</td>
      <td>In January 2020, concerns were raised about th...</td>
      <td>369</td>
    </tr>
    <tr>
      <th>3</th>
      <td>2020 Summer Olympics</td>
      <td>Qualifying event cancellation and postponement</td>
      <td>Concerns about the pandemic began to affect qu...</td>
      <td>298</td>
    </tr>
    <tr>
      <th>4</th>
      <td>2020 Summer Olympics</td>
      <td>Effect on doping tests</td>
      <td>Mandatory doping tests were being severely res...</td>
      <td>163</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "141f71a5-7b4d-46f8-8944-24611dced5c6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "378c6165-3248-437f-bbb6-bcf14e8f390e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3a8a6133f9a91f6c5d7e64f055a62a395bae57425cd2b65daa6f49d3ba186d16", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9c470077-244b-4407-8fd0-440755f6f067", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-1-collect-data.ipynb", "title": "1. Collect Wikipedia data about Olympic Games 2020", "tokens": 2914, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "45e8d34618c0b72b77bf6765466aae3b4a5a0f33b79d941f9f907544b65107d1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10518, "end_char_idx": 12062, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Save the section dataset
We will save the section dataset, for the [next notebook](olympics-2-create-qa.ipynb)


```python
df.to_csv('olympics-data/olympics_sections.csv', index=False)
```

## 1.3 (Optional) Exploring the data 


```python
df.title.value_counts().head()
```




    Concerns and controversies at the 2020 Summer Olympics    51
    United States at the 2020 Summer Olympics                 46
    Great Britain at the 2020 Summer Olympics                 42
    Canada at the 2020 Summer Olympics                        39
    Olympic Games                                             39
    Name: title, dtype: int64



There appear to be winter and summer Olympics 2020. We chose to leave a little ambiguity and noise in the dataset, even though we were interested in only Summer Olympics 2020.


```python
df.title.str.contains('Summer').value_counts()
```




    True     3567
    False     305
    Name: title, dtype: int64




```python
df.title.str.contains('Winter').value_counts()
```




    False    3774
    True       98
    Name: title, dtype: int64




```python
import pandas as pd
from matplotlib import pyplot as plt

df = pd.read_csv('olympics-data/olympics_sections.csv')
df[['tokens']].hist()
# add axis descriptions and title
plt.xlabel('Number of tokens')
plt.ylabel('Number of Wikipedia sections')
plt.title('Distribution of number of tokens in Wikipedia sections')
plt.show()
```


    
![png](output_15_0.png)
    


We can see that the majority of section are fairly short (less than 500 tokens).


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb


key: _node_content
string_value:
{"id_": "7b2011dc-fcb3-491b-a21e-aa8e82dffd55", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "da1fc694-7ce8-490b-8439-93f1135f940e", "node_type": "1", "metadata": {}, "hash": "f898e720fef8a88b50fc010a9994ac9429bfb03f2224422bc1dda50fe6814fe5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 1397, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<span style="color:orange; font-weight:bold">Note: To answer questions based on text documents, we recommend the procedure in <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering using Embeddings</a>. Some of the code below may rely on <a href="https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints">deprecated API endpoints</a>.</span>

# 2. Creating a synthetic Q&A dataset
We use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), a model specialized in following instructions, to create questions based on the given context. Then we also use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta) to answer those questions, given the same context. 

This is expensive, and will also take a long time, as we call the davinci engine for each section. You can simply download the final dataset instead.

We're using the dataset created using the [previous notebook](olympics-1-collect-data.ipynb)

## 2.1 Read in the data, and create a context
Create a context by concatenating the title, the heading and the content of that section


```python
import pandas as pd
df = pd.read_csv('olympics-data/olympics_sections.csv')
df['context'] = df.title + "\n" + df.heading + "\n\n" + df.content
df.head()
```


key: _node_content
string_value:
{"id_": "da1fc694-7ce8-490b-8439-93f1135f940e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7b2011dc-fcb3-491b-a21e-aa8e82dffd55", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "6a735d22633015e25d9e78263ede3d5beb57dc8604e491531b877509f2125c6b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "04507245-d7d8-4aaa-97f2-9cb5ec772602", "node_type": "1", "metadata": {}, "hash": "bd95ab2fbdc31bed7ed03497313e3fea2bd8f3253b0bb22457cffb08619c6ff3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3215, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<span style="color:orange; font-weight:bold">Note: To answer questions based on text documents, we recommend the procedure in <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering using Embeddings</a>. Some of the code below may rely on <a href="https://github.com/openai/openai-cookbook/tree/main/transition_guides_for_deprecated_API_endpoints">deprecated API endpoints</a>.</span>

# 2. Creating a synthetic Q&A dataset
We use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), a model specialized in following instructions, to create questions based on the given context. Then we also use [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta) to answer those questions, given the same context. 

This is expensive, and will also take a long time, as we call the davinci engine for each section. You can simply download the final dataset instead.

We're using the dataset created using the [previous notebook](olympics-1-collect-data.ipynb)

## 2.1 Read in the data, and create a context
Create a context by concatenating the title, the heading and the content of that section


```python
import pandas as pd
df = pd.read_csv('olympics-data/olympics_sections.csv')
df['context'] = df.title + "\n" + df.heading + "\n\n" + df.content
df.head()
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>heading</th>
      <th>content</th>
      <th>tokens</th>
      <th>context</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>2020 Summer Olympics</td>
      <td>Summary</td>
      <td>The 2020 Summer Olympics (Japanese: 2020年夏季オリン...</td>
      <td>713</td>
      <td>2020 Summer Olympics\nSummary\n\nThe 2020 Summ...</td>
    </tr>
    <tr>
      <th>1</th>
      <td>2020 Summer Olympics</td>
      <td>Host city selection</td>
      <td>The International Olympic Committee (IOC) vote...</td>
      <td>126</td>
      <td>2020 Summer Olympics\nHost city selection\n\nT...</td>
    </tr>
    <tr>
      <th>2</th>
      <td>2020 Summer Olympics</td>
      <td>Impact of the COVID-19 pandemic</td>
      <td>In January 2020, concerns were raised about th...</td>
      <td>369</td>
      <td>2020 Summer Olympics\nImpact of the COVID-19 p...</td>
    </tr>
    <tr>
      <th>3</th>
      <td>2020 Summer Olympics</td>
      <td>Qualifying event cancellation and postponement</td>
      <td>Concerns about the pandemic began to affect qu...</td>
      <td>298</td>
      <td>2020 Summer Olympics\nQualifying event cancell...</td>
    </tr>
    <tr>
      <th>4</th>
      <td>2020 Summer Olympics</td>
      <td>Effect on doping tests</td>
      <td>Mandatory doping tests were being severely res...</td>
      <td>163</td>
      <td>2020 Summer Olympics\nEffect on doping tests\n...</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "04507245-d7d8-4aaa-97f2-9cb5ec772602", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "da1fc694-7ce8-490b-8439-93f1135f940e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d60d944bc36e857d7f2b93a57e284652d26f6d0958f33c2cb69352f82097de9d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6a645267-4b03-4f0b-94c5-ee16c5dd3346", "node_type": "1", "metadata": {}, "hash": "37ff249c3e5c99e7e601e25d040626d3ec1872124757ebe04272156e33ff6db4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3219, "end_char_idx": 5081, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 2.2 Create questions based on the context
Use davinci-instruct to generate a number of plausible questions relating to the Wikipedia section contents.

Note: We have used temperature=0, but it may be beneficial to experiment with a higher temperature to get a higher diversity of questions.

<span style="color:orange; font-weight:bold">WARNING: This step will last a long time, and consume a lot of tokens, as it calls davinci-instruct for every section to generate a number of questions.</span>


```python
from openai import OpenAI

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

def get_questions(context):
    try:
        response = client.chat.completions.create(model="davinci-instruct-beta-v3",
        prompt=f"Write questions based on the text below\n\nText: {context}\n\nQuestions:\n1.",
        temperature=0,
        max_tokens=257,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0,
        stop=["\n\n"])
        return response.choices[0].text
    except:
        return ""


df['questions']= df.context.apply(get_questions)
df['questions'] = "1." + df.questions
print(df[['questions']].values[0][0])
```

    1. What is the 2020 Summer Olympics?
    2. When did the 2020 Summer Olympics take place?
    3. Who won the most medals at the 2020 Summer Olympics?
    4. Who won the most gold medals at the 2020 Summer Olympics?
    5. Who won the most medals at the 2020 Summer Olympics?


The prompt is designed to generate a number of questions. Example questions above were generated based on the summary section of the 2020 Summer Olympics page.

We can observe that the questions 3 and 5 above repeat. Sometimes the generated questions could be ambiguous without the context. We will show that even despite these limitations we can create a successful model.


key: _node_content
string_value:
{"id_": "6a645267-4b03-4f0b-94c5-ee16c5dd3346", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "04507245-d7d8-4aaa-97f2-9cb5ec772602", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "cf436fcf584059b642229f24e698316541da77eb6f6a154772b6e84162cf8e08", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a2560c6b-5523-47d9-96f8-e5126a9bfad7", "node_type": "1", "metadata": {}, "hash": "e0726c13f839c826ae16182175605a6a17f0059ca5f8eeac18e89949dfc481ce", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3721, "end_char_idx": 8249, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
from openai import OpenAI

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

def get_questions(context):
    try:
        response = client.chat.completions.create(model="davinci-instruct-beta-v3",
        prompt=f"Write questions based on the text below\n\nText: {context}\n\nQuestions:\n1.",
        temperature=0,
        max_tokens=257,
        top_p=1,
        frequency_penalty=0,
        presence_penalty=0,
        stop=["\n\n"])
        return response.choices[0].text
    except:
        return ""


df['questions']= df.context.apply(get_questions)
df['questions'] = "1." + df.questions
print(df[['questions']].values[0][0])
```

    1. What is the 2020 Summer Olympics?
    2. When did the 2020 Summer Olympics take place?
    3. Who won the most medals at the 2020 Summer Olympics?
    4. Who won the most gold medals at the 2020 Summer Olympics?
    5. Who won the most medals at the 2020 Summer Olympics?


The prompt is designed to generate a number of questions. Example questions above were generated based on the summary section of the 2020 Summer Olympics page.

We can observe that the questions 3 and 5 above repeat. Sometimes the generated questions could be ambiguous without the context. We will show that even despite these limitations we can create a successful model.


```python
print(df.content.values[0])
```

    The 2020 Summer Olympics (Japanese: 2020年夏季オリンピック, Hepburn: Nisen Nijū-nen Kaki Orinpikku), officially the Games of the XXXII Olympiad (第三十二回オリンピック競技大会, Dai Sanjūni-kai Orinpikku Kyōgi Taikai) and branded as Tokyo 2020 (東京2020, Tōkyō Nii Zero Nii Zero), was an international multi-sport event held from 23 July to 8 August 2021 in Tokyo, Japan, with some preliminary events that began on 21 July.
    Tokyo was selected as the host city during the 125th IOC Session in Buenos Aires, Argentina, on 7 September 2013. Originally scheduled to take place from 24 July to 9 August 2020, the event was postponed to 2021 in March 2020 as a result of the COVID-19 pandemic, the first such instance in the history of the Olympic Games (previous games had been cancelled but not rescheduled). However, the event retained the Tokyo 2020 name for marketing and branding purposes. It was largely held behind closed doors with no public spectators permitted due to the declaration of a state of emergency in the Greater Tokyo Area in response to the pandemic. The Summer Paralympics were held between 24 August and 5 September 2021, 16 days after the completion of the Olympics.The 2020 Games were the fourth Olympic Games to be held in Japan, following the Tokyo 1964 (Summer), Sapporo 1972 (Winter) and Nagano 1998 (Winter) games. Tokyo is the first city in Asia to hold the Summer Games twice. The 2020 Games were the second of three consecutive Olympics to be held in East Asia, following the 2018 Winter Olympics in Pyeongchang, South Korea and preceding the 2022 Winter Olympics in Beijing, China.
    New events were introduced in existing sports for 2020, including 3x3 basketball, freestyle BMX and mixed gender team events in a number of existing sports, as well as the return of madison cycling for men and an introduction of the same event for women. New IOC policies also allowed the host organizing committee to add new sports to the Olympic program for just one Games. The disciplines added by the Japanese Olympic Committee were baseball and softball, karate, sport climbing, surfing and skateboarding, the last four of which made their Olympic debuts, and the last three of which will remain on the Olympic program.The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88). Host nation Japan finished third, setting a record for the most gold medals and total medals ever won by their delegation at an Olympic Games with 27 and 58. Great Britain finished fourth, with a total of 22 gold and 65 medals, becoming the first nation at the Summer Olympics to increase or equal their total medals won in the two Games subsequent to hosting them. The Russian delegation competing as the ROC (not to be confused with the Republic of China (Taiwan) which competed as Chinese Taipei, not ROC) finished fifth with 20 gold medals and third in the overall medal count, with 71 medals. Bermuda, the Philippines and Qatar won their first-ever Olympic gold medals. Burkina Faso, San Marino and Turkmenistan won their first-ever Olympic medals.


key: _node_content
string_value:
{"id_": "a2560c6b-5523-47d9-96f8-e5126a9bfad7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6a645267-4b03-4f0b-94c5-ee16c5dd3346", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d548f2cc12df3e3b7a6eba71d9b39b522601e72923332c318020f7573b637757", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8e892a85-b8dc-4053-8bd5-1e380b49a579", "node_type": "1", "metadata": {}, "hash": "e2d82bcea0391beeace8fa174d127e5a91d9890e89be86df52aab2755179c96d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8252, "end_char_idx": 11280, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 2.3 Create answers based on the context
Use davinci-instruct to answer the questions given the relevant Wikipedia section contents

Note: We have used temperature=0, but it may be beneficial to experiment with a higher temperature to get a higher diversity of questions.

<span style="color:orange">**WARNING: This step will last a long time, and consume a lot of tokens, as it calls davinci-instruct for every section to answer all the questions.**</span>


```python
def get_answers(row):
    try:
        response = client.chat.completions.create(
            engine="davinci-instruct-beta-v3",
            prompt=f"Write answer based on the text below\n\nText: {row.context}\n\nQuestions:\n{row.questions}\n\nAnswers:\n1.",
            temperature=0,
            max_tokens=257,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0
        )
        return response.choices[0].text
    except Exception as e:
        print (e)
        return ""


df['answers']= df.apply(get_answers, axis=1)
df['answers'] = "1." + df.answers
df = df.dropna().reset_index().drop('index',axis=1)
print(df[['answers']].values[0][0])
```

    1. The 2020 Summer Olympics is an international multi-sport event held from 23 July to 8 August 2021 in Tokyo, Japan.
    2. The 2020 Summer Olympics took place from 23 July to 8 August 2021.
    3. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).
    4. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).
    5. The United States topped the medal count by both total golds (39) and total medals (113), with China finishing second by both respects (38 and 88).


These are the answers to the questions above based on the context around the host city selection. 

We can see that answers 3-5 contain the correct answer, but instead of answering the question directly, the answer is a verbatim extraction. Despite these occasional lower quality answers, we will show that the model can learn the task reasonably well, given a high number of examples.

## 2.4 Save the Olympics Q&A dataset based on Wikipedia sections
We save the file for use in the [next notebook](olympics-3-train-qa.ipynb)


```python
df.to_csv('olympics-data/olympics_qa.csv', index=False)
```

## 2.5 Search file (DEPRECATED)
We create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.

<span style="color:orange; font-weight:bold">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering Guide</a> for a search implementation using the embeddings</span>


key: _node_content
string_value:
{"id_": "8e892a85-b8dc-4053-8bd5-1e380b49a579", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a2560c6b-5523-47d9-96f8-e5126a9bfad7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "9cf622602c2607d3253a740f647ea1035019f56db5c1ef51d6068c11f8461d8c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e757fc04-90c9-4668-97a7-fd33fe956cee", "node_type": "1", "metadata": {}, "hash": "b26160b2fc3dd4b93c22d0e38b68ab6b5ca5deab448fedc70b44d3852594faa9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10073, "end_char_idx": 12028, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
These are the answers to the questions above based on the context around the host city selection. 

We can see that answers 3-5 contain the correct answer, but instead of answering the question directly, the answer is a verbatim extraction. Despite these occasional lower quality answers, we will show that the model can learn the task reasonably well, given a high number of examples.

## 2.4 Save the Olympics Q&A dataset based on Wikipedia sections
We save the file for use in the [next notebook](olympics-3-train-qa.ipynb)


```python
df.to_csv('olympics-data/olympics_qa.csv', index=False)
```

## 2.5 Search file (DEPRECATED)
We create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.

<span style="color:orange; font-weight:bold">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering Guide</a> for a search implementation using the embeddings</span>



```python
df = df[df.tokens<2000]
df[['context', 'tokens']].rename(columns={'context':'text','tokens':'metadata'}).to_json('olympics-data/olympics_search.jsonl', orient='records', lines=True)

search_file = client.files.create(
  file=open("olympics-data/olympics_search.jsonl"),
  purpose='search'
)
olympics_search_fileid = search_file['id']
```

## 2.6 Answer questions based on the context provided

We will use a simple implementation of the answers endpoint. This works by simply using the [/search endpoint](https://beta.openai.com/docs/api-reference/searches), which searches over an indexed file to obtain the relevant sections which can be included in the context, following by a question and answering prompt given a specified model.


key: _node_content
string_value:
{"id_": "e757fc04-90c9-4668-97a7-fd33fe956cee", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8e892a85-b8dc-4053-8bd5-1e380b49a579", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "7d97047947889f466f9a82056ec82541870568e1b73f65bb2529b4709256e370", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6e47d874-a87a-4ac7-866c-4546ce82ad8c", "node_type": "1", "metadata": {}, "hash": "2f0267626d95ae41aff474e162dacff7692b380bdd626554cae4e525be23dca0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10602, "end_char_idx": 14076, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
df.to_csv('olympics-data/olympics_qa.csv', index=False)
```

## 2.5 Search file (DEPRECATED)
We create a search file ([API reference](https://beta.openai.com/docs/api-reference/files/list)), which can be used to retrieve the relevant context when a question is asked.

<span style="color:orange; font-weight:bold">DEPRECATED: The /search endpoint is deprecated in favour of using embeddings. Embeddings are cheaper, faster and can support a better search experience. See <a href="https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_embeddings.ipynb">Question Answering Guide</a> for a search implementation using the embeddings</span>



```python
df = df[df.tokens<2000]
df[['context', 'tokens']].rename(columns={'context':'text','tokens':'metadata'}).to_json('olympics-data/olympics_search.jsonl', orient='records', lines=True)

search_file = client.files.create(
  file=open("olympics-data/olympics_search.jsonl"),
  purpose='search'
)
olympics_search_fileid = search_file['id']
```

## 2.6 Answer questions based on the context provided

We will use a simple implementation of the answers endpoint. This works by simply using the [/search endpoint](https://beta.openai.com/docs/api-reference/searches), which searches over an indexed file to obtain the relevant sections which can be included in the context, following by a question and answering prompt given a specified model.


```python
from answers_with_ft import create_context, answer_question
print(create_context("Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?", olympics_search_fileid, max_len=400))
```

    Athletics at the 2020 Summer Olympics – Women's 4 × 100 metres relay
    Summary
    
    The women's 4 × 100 metres relay event at the 2020 Summer Olympics took place on 5 and 6 August 2021 at the Japan National Stadium. There were 16 competing relay teams, with each team having 5 members from which 4 were selected in each round.
    
    ###
    
    Athletics at the 2020 Summer Olympics – Men's 4 × 100 metres relay
    Qualification
    
    National Olympic Committees (NOCs) could qualify one relay team in one of three following ways:
    The top 8 NOCs at the 2019 World Athletics Championships qualified a relay team.
    The top 8 NOCs at the 2021 World Athletics Relays qualified a relay team.
    Where an NOC placed in the top 8 at both the 2019 World Championships and the 2021 World Relays, the quota place was allocated to the world top list as of 29 June 2021. In this case, 4 teams did so, so there are 4 places available through the world rankings.A total of five athletes may be entered for a relay team. Should a NOC have also entered individual athletes in the corresponding individual event (100 m), the entered individual athletes must be included in the total of five (5) athletes entered for the relay event. In addition of five, NOCs can nominate a maximum of one alternate athlete for each team.
    The qualifying period was originally from 1 May 2019 to 29 June 2020. Due to the COVID-19 pandemic, the period was suspended from 6 April 2020 to 30 November 2020, with the end date extended to 29 June 2021. The qualifying time standards could be obtained in various meets during the given period that have the approval of the IAAF. Both indoor and outdoor meets are eligible. The most recent Area Championships may be counted in the ranking, even if not during the qualifying period.


key: _node_content
string_value:
{"id_": "6e47d874-a87a-4ac7-866c-4546ce82ad8c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e757fc04-90c9-4668-97a7-fd33fe956cee", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "9fcd86934fc9d29d04399771233349e9a941a5b1f6702df6b8936e09b15b6d50", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d9728585-2a7c-4b79-825a-9b2aa00529ee", "node_type": "1", "metadata": {}, "hash": "e162a1ea5a2e13a63d47a7a226af8464a6db998ee2dfe66f4371b7ccdf6096d5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14080, "end_char_idx": 15448, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
answer_question(olympics_search_fileid, "davinci-instruct-beta-v3", 
            "Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?")
```




    ' Japan National Stadium'



After we fine-tune the model for Q&A we'll be able to use it instead of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), to obtain better answers when the question can't be answered based on the context. We see a downside of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), which always attempts to answer the question, regardless of the relevant context being present or not. (Note the second question is asking about a future event, set in 2024.)


```python
answer_question(olympics_search_fileid, "davinci-instruct-beta-v3", 
            "Where did women's 4 x 100 metres relay event take place during the 2048 Summer Olympics?", max_len=1000)
```




    ' Japan National Stadium'



We can see that davinci has a tendency to answer the question, even if the question can't be answered given the context provided. Note the question asked regarding 2048 Summer Olympics, which didn't happen yet, and the retrieved content has only returned results for 2020.

## 2.7 (Optional) Investigation into how likely the search endpoint is to return the relevant context


key: _node_content
string_value:
{"id_": "d9728585-2a7c-4b79-825a-9b2aa00529ee", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6e47d874-a87a-4ac7-866c-4546ce82ad8c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "89c66e22e494442c292c09d1c4d22d57be09fb649104880b24a7cbb2f5d62f8e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2d9980bb-36f6-4369-9f3d-452b096bcc2f", "node_type": "1", "metadata": {}, "hash": "c69dba5798852c1c363dc15b6b742c21b38c995a565652d205b7fd4e889cb418", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14080, "end_char_idx": 17343, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
answer_question(olympics_search_fileid, "davinci-instruct-beta-v3", 
            "Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?")
```




    ' Japan National Stadium'



After we fine-tune the model for Q&A we'll be able to use it instead of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), to obtain better answers when the question can't be answered based on the context. We see a downside of [`davinci-instruct-beta-v3`](https://beta.openai.com/docs/engines/instruct-series-beta), which always attempts to answer the question, regardless of the relevant context being present or not. (Note the second question is asking about a future event, set in 2024.)


```python
answer_question(olympics_search_fileid, "davinci-instruct-beta-v3", 
            "Where did women's 4 x 100 metres relay event take place during the 2048 Summer Olympics?", max_len=1000)
```




    ' Japan National Stadium'



We can see that davinci has a tendency to answer the question, even if the question can't be answered given the context provided. Note the question asked regarding 2048 Summer Olympics, which didn't happen yet, and the retrieved content has only returned results for 2020.

## 2.7 (Optional) Investigation into how likely the search endpoint is to return the relevant context


```python
def check_context(title, heading, question, max_len=1800, search_model='ada', max_rerank=10):
    """
    Evaluate the performance of the search model in retrieving the correct context

    Parameters
    ----------
    title: str
        The title of the Wikipedia page
    heading: str
        The heading of the Wikipedia section
    qusetion: str
        The question
    max_len: int
        The maximum length of the context
    search_model: str
        The search model to use - `ada` is most cost effective
    max_rerank: int
        The maximum number of reranking documents to use the search model on

    Returns
    -------
    rank: int
        The rank of the correct context
    token_length: int
        The number of tokens needed to obtain the correct context
    """
    
    try:
        # TODO: openai.Engine(search_model) is deprecated
        results = openai.Engine(search_model).search(
            search_model=search_model, 
            query=question, 
            max_rerank=max_rerank,
            file=olympics_search_fileid,
            return_metadata=True
        )
        index=-1
        returns = []
        cur_len = 0
        for result in results['data']:
            cur_len += int(result['metadata']) + 4 # we add 4 tokens for the separator `\n\n###\n\n`
            if cur_len > max_len:
                break
            returns.append(result['text'])
            res = result['text'].split('\n')
            if res[0] == title and res[1] == heading:
                index = len(returns) - 1
                break
        return index, cur_len
    except Exception as e:
        #print (e)
        return []
print(check_context("Athletics at the 2020 Summer Olympics – Women's 4 × 100 metres relay", "Summary", "Where did women's 4 x 100 metres relay event take place during the 2020 Summer Olympics?", max_len=10000))
```

    (0, 58)


key: _node_content
string_value:
{"id_": "2d9980bb-36f6-4369-9f3d-452b096bcc2f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d9728585-2a7c-4b79-825a-9b2aa00529ee", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "8bd676687634852a393618a379893324df0f97d472cc602aa9022ca3cbe3f643", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "cd16134d-c389-4283-8c77-018997a236e9", "node_type": "1", "metadata": {}, "hash": "068a6b7ab450d88ef40c83ff16b16f5c60058205e74bc92522d5612de1a2dfa8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 17346, "end_char_idx": 20158, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
We utilize the generated questions based on context to estimate how often we can retrieve the original context. These questions are noisy, so this is not a perfect estimate.

Our questions and answers are prefixed with numbered bullet points, however due to the way they were generated, they are missing the first number, hence we add "1." to the list of questions (and answers).

We calculate the rank of the section retrieved using ada search, and the number of tokens in the context needed to retrieve the relevant section in full.


```python
ada_results = df.apply(lambda x: [
                    check_context( x.title, 
                                   x.heading, 
                                   q[3:],     # remove the number prefix
                                   max_len=1000000, # set a large number to get the full context 
                                   search_model='ada', 
                                   max_rerank=200,
                                 ) 
                    for q in (x.questions).split('\n') # split the questions
                    if len(q) >10 # remove the empty questions
                ], axis=1)
ada_results.head()
```




    0    [(132, 27104), (-1, 22939), (8, 2151), (2, 121...
    1    [(4, 1737), (0, 130), (8, 744), (96, 17208), (...
    2          [(0, 373), (0, 373), (-1, 40610), (1, 570)]
    3            [(0, 302), (0, 302), (5, 968), (8, 1425)]
    4                      [(0, 167), (0, 167), (2, 1442)]
    Name: ada, dtype: object




```python
out = pd.concat([ada_results], axis=1)
out.columns = ['ada']
out.to_csv('olympics-data/search_engine_results.csv')
```


```python
def expand_lists(out):
    """
    Expand a pandas series containing lists into a series, where each list element becomes a value on its own

    Input is a row per paragraph, which has multiple questions
    Output is a row per question
    """
    cols = [pd.DataFrame(out[name].tolist()).stack().reset_index(level=1, drop=True).rename(name) for name in out.columns] 
    return pd.concat(cols, axis=1)

out_expanded = expand_lists(out)
out_expanded['rank'] = out_expanded.ada.apply(lambda x: x[0] if x != [] else -2)
out_expanded['tokens'] = out_expanded.ada.apply(lambda x: x[1] if x != [] else -2)

```


```python
within_2k = (out_expanded.tokens < 2000).mean()
print(f"{within_2k*100:.1f}% of relevant paragraphs are retrieved within the first 2k tokens")
```

    74.3% of relevant paragraphs are retrieved within the first 2k tokens


The relevant context can be obtained 74% of the time on this dataset


```python
outside_200 = (out_expanded['rank'] == -1).mean()
print(f"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results")
```

    7.4% of relevant paragraphs are not retrieved within the first 200 results


key: _node_content
string_value:
{"id_": "cd16134d-c389-4283-8c77-018997a236e9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2d9980bb-36f6-4369-9f3d-452b096bcc2f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "b2999c2b87a8b26ccbd9eb0d69a8327acc499eb67f160ce82bd6ba16f315601b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a251b171-c8d1-4fe5-b55b-8dc2338f4b0d", "node_type": "1", "metadata": {}, "hash": "062d44d0693558cf08d89eb8e656f1b0c99a05740d3b694619f9f64e1b71fe6e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 18856, "end_char_idx": 21286, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
out = pd.concat([ada_results], axis=1)
out.columns = ['ada']
out.to_csv('olympics-data/search_engine_results.csv')
```


```python
def expand_lists(out):
    """
    Expand a pandas series containing lists into a series, where each list element becomes a value on its own

    Input is a row per paragraph, which has multiple questions
    Output is a row per question
    """
    cols = [pd.DataFrame(out[name].tolist()).stack().reset_index(level=1, drop=True).rename(name) for name in out.columns] 
    return pd.concat(cols, axis=1)

out_expanded = expand_lists(out)
out_expanded['rank'] = out_expanded.ada.apply(lambda x: x[0] if x != [] else -2)
out_expanded['tokens'] = out_expanded.ada.apply(lambda x: x[1] if x != [] else -2)

```


```python
within_2k = (out_expanded.tokens < 2000).mean()
print(f"{within_2k*100:.1f}% of relevant paragraphs are retrieved within the first 2k tokens")
```

    74.3% of relevant paragraphs are retrieved within the first 2k tokens


The relevant context can be obtained 74% of the time on this dataset


```python
outside_200 = (out_expanded['rank'] == -1).mean()
print(f"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results")
```

    7.4% of relevant paragraphs are not retrieved within the first 200 results


7.4% of the time, this is due to the keyword search part of the search algorithm not retrieving the relevant context within the first 200 results.
18.3% of the time this is due to the semantic search not placing the relevant context within the first 2000 tokens.


```python
import matplotlib.pyplot as plt

# plot a histogram, and add axis descriptions and title
out_expanded[(out_expanded['rank'] >=0)&(out_expanded['rank'] <30)]['rank'].hist(bins=29)
plt.xlabel('rank')
plt.ylabel('count')
plt.title('Histogram of ranks of retrieved paragraphs')
plt.show()
```


    
![png](output_31_0.png)
    



```python
out_expanded[(out_expanded.tokens>=0)&(out_expanded.tokens < 2000)]['tokens'].hist(bins=29)
plt.xlabel('tokens')
plt.ylabel('count')
plt.title('Histogram of the number of minimum tokens needed')
plt.show()
```


    
![png](output_32_0.png)
    


We can observe that the context is most likely to be returned as one of the first results, and most likely to be returned within the first 200-500 tokens.


```python
# normalized value_counts
out_expanded['rank'].value_counts(normalize=True).sort_index()[:13]
```


key: _node_content
string_value:
{"id_": "a251b171-c8d1-4fe5-b55b-8dc2338f4b0d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "02600b84-105a-4d25-93bb-1f1eaf52d43a", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1dace799e477d710c8a5f5e07dd9eb059b8ce0b711356d9bc729f127541dac5b", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "cd16134d-c389-4283-8c77-018997a236e9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/olympics-2-create-qa.ipynb", "title": "2. Creating a synthetic Q&A dataset", "tokens": 5416, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "a00993fda6ae1f8d6c8c7836490516b49574f6ec100efc9b34bfb90cb2e562b2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 19841, "end_char_idx": 21708, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The relevant context can be obtained 74% of the time on this dataset


```python
outside_200 = (out_expanded['rank'] == -1).mean()
print(f"{outside_200*100:.1f}% of relevant paragraphs are not retrieved within the first 200 results")
```

    7.4% of relevant paragraphs are not retrieved within the first 200 results


7.4% of the time, this is due to the keyword search part of the search algorithm not retrieving the relevant context within the first 200 results.
18.3% of the time this is due to the semantic search not placing the relevant context within the first 2000 tokens.


```python
import matplotlib.pyplot as plt

# plot a histogram, and add axis descriptions and title
out_expanded[(out_expanded['rank'] >=0)&(out_expanded['rank'] <30)]['rank'].hist(bins=29)
plt.xlabel('rank')
plt.ylabel('count')
plt.title('Histogram of ranks of retrieved paragraphs')
plt.show()
```


    
![png](output_31_0.png)
    



```python
out_expanded[(out_expanded.tokens>=0)&(out_expanded.tokens < 2000)]['tokens'].hist(bins=29)
plt.xlabel('tokens')
plt.ylabel('count')
plt.title('Histogram of the number of minimum tokens needed')
plt.show()
```


    
![png](output_32_0.png)
    


We can observe that the context is most likely to be returned as one of the first results, and most likely to be returned within the first 200-500 tokens.


```python
# normalized value_counts
out_expanded['rank'].value_counts(normalize=True).sort_index()[:13]
```




    -2     0.000063
    -1     0.074428
     0     0.453420
     1     0.089515
     2     0.047146
     3     0.032437
     4     0.024139
     5     0.019676
     6     0.015967
     7     0.013452
     8     0.011189
     9     0.009869
     10    0.009178
    Name: rank, dtype: float64



probabilities of the relevant context being returned at each rank. (-2 means a processing error, -1 means the rank is >200)


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb


key: title
string_value:
Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning


key: _node_content
string_value:
{"id_": "a34ff3dc-11a1-4928-93fe-c7d3c42debf0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ebfa7d19-69f9-43b7-9d2f-a8d94943f71a", "node_type": "1", "metadata": {}, "hash": "bd6309d85754359747104e4d5a7c9b5f0a223073ce49b3d0e67a24c91449ebb3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3009, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning

The aim of this notebook is to walk through a comprehensive example of how to fine-tune OpenAI models for Retrieval Augmented Generation (RAG). 

We will also be integrating Qdrant and Few-Shot Learning to boost the model's performance and reduce hallucinations. This could serve as a practical guide for ML practitioners, data scientists, and AI Engineers interested in leveraging the power of OpenAI models for specific use-cases. 🤩

## Why should you read this blog?

You want to learn how to 
- [Fine-tune OpenAI models](https://platform.openai.com/docs/guides/fine-tuning/) for specific use-cases
- Use [Qdrant](https://qdrant.tech/documentation/) to improve the performance of your RAG model
- Use fine-tuning to improve the correctness of your RAG model and reduce hallucinations

To begin, we've selected a dataset where we've a guarantee that the retrieval is perfect. We've selected a subset of the [SQuAD](https://rajpurkar.github.io/SQuAD-explorer/) dataset, which is a collection of questions and answers about Wikipedia articles. We've also included samples where the answer is not present in the context, to demonstrate how RAG handles this case.

## Table of Contents
1. Setting up the Environment

### Section A: Zero-Shot Learning
2. Data Preparation: SQuADv2 Dataset
3. Answering using Base gpt-3.5-turbo-0613 model
4. Fine-tuning and Answering using Fine-tuned model
5. **Evaluation**: How well does the model perform?

### Section B: Few-Shot Learning

6. Using Qdrant to Improve RAG Prompt
7. Fine-Tuning OpenAI Model with Qdrant
8. Evaluation

9. **Conclusion**
    - Aggregate Results
    - Observations

## Terms, Definitions, and References

**Retrieval Augmented Generation (RAG)?**
The phrase Retrieval Augmented Generation (RAG) comes from a [recent paper](https://arxiv.org/abs/2005.11401) by Lewis et al. from Facebook AI. The idea is to use a pre-trained language model (LM) to generate text, but to use a separate retrieval system to find relevant documents to condition the LM on. 

**What is Qdrant?**
Qdrant is an open-source vector search engine that allows you to search for similar vectors in a large dataset. It is built in Rust and here we'll use the Python client to interact with it. This is the Retrieval part of RAG.

**What is Few-Shot Learning?**
Few-shot learning is a type of machine learning where the model is "improved" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.

**What is Zero-Shot Learning?**
Zero-shot learning is a type of machine learning where the model is "improved" via training or fine-tuning without any dataset specific information. 

**What is Fine-Tuning?**
Fine-tuning is a type of machine learning where the model is "improved" via training or fine-tuning on a small amount of data.


key: _node_content
string_value:
{"id_": "ebfa7d19-69f9-43b7-9d2f-a8d94943f71a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a34ff3dc-11a1-4928-93fe-c7d3c42debf0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "61118c1d33336410355df7f7601489f39754bb03adc12f3fb442d2a6a79312b9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "accd346b-7b95-4fe1-8845-4877c3c0f272", "node_type": "1", "metadata": {}, "hash": "19882e9be2e5a9df2b233c2312f3da3ffc7779d8b9286d05594df5ddda2a7578", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1442, "end_char_idx": 4059, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Fine-tuning and Answering using Fine-tuned model
5. **Evaluation**: How well does the model perform?

### Section B: Few-Shot Learning

6. Using Qdrant to Improve RAG Prompt
7. Fine-Tuning OpenAI Model with Qdrant
8. Evaluation

9. **Conclusion**
    - Aggregate Results
    - Observations

## Terms, Definitions, and References

**Retrieval Augmented Generation (RAG)?**
The phrase Retrieval Augmented Generation (RAG) comes from a [recent paper](https://arxiv.org/abs/2005.11401) by Lewis et al. from Facebook AI. The idea is to use a pre-trained language model (LM) to generate text, but to use a separate retrieval system to find relevant documents to condition the LM on. 

**What is Qdrant?**
Qdrant is an open-source vector search engine that allows you to search for similar vectors in a large dataset. It is built in Rust and here we'll use the Python client to interact with it. This is the Retrieval part of RAG.

**What is Few-Shot Learning?**
Few-shot learning is a type of machine learning where the model is "improved" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.

**What is Zero-Shot Learning?**
Zero-shot learning is a type of machine learning where the model is "improved" via training or fine-tuning without any dataset specific information. 

**What is Fine-Tuning?**
Fine-tuning is a type of machine learning where the model is "improved" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. The LLM is what makes the Generation part of RAG.

## 1. Setting Up the Environment

### Install and Import Dependencies


```python
!pip install pandas openai tqdm tenacity scikit-learn tiktoken python-dotenv seaborn --upgrade --quiet
```


```python
import json
import os
import time

import pandas as pd
from openai import OpenAI
import tiktoken
import seaborn as sns
from tenacity import retry, wait_exponential
from tqdm import tqdm
from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix

import warnings
warnings.filterwarnings('ignore')

tqdm.pandas()

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

```

### Set your keys
Get your OpenAI keys [here](https://platform.openai.com/account/api-keys) and Qdrant keys after making a free cluster [here](https://cloud.qdrant.io/login).


key: _node_content
string_value:
{"id_": "accd346b-7b95-4fe1-8845-4877c3c0f272", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ebfa7d19-69f9-43b7-9d2f-a8d94943f71a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "42613a99e181198e4c06b400f4efb7b57eea9a7720aea409e94e3878bf080343", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2741944f-30bd-4890-8462-9b40f7b7d4b9", "node_type": "1", "metadata": {}, "hash": "7afdd8b85611d4f227050ddcbfda91240c9004a269ef33b06ba69acbac0892b5", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2531, "end_char_idx": 4967, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. This is the Augmented part of RAG.

**What is Zero-Shot Learning?**
Zero-shot learning is a type of machine learning where the model is "improved" via training or fine-tuning without any dataset specific information. 

**What is Fine-Tuning?**
Fine-tuning is a type of machine learning where the model is "improved" via training or fine-tuning on a small amount of data. In this case, we'll use it to fine-tune the RAG model on a small number of examples from the SQuAD dataset. The LLM is what makes the Generation part of RAG.

## 1. Setting Up the Environment

### Install and Import Dependencies


```python
!pip install pandas openai tqdm tenacity scikit-learn tiktoken python-dotenv seaborn --upgrade --quiet
```


```python
import json
import os
import time

import pandas as pd
from openai import OpenAI
import tiktoken
import seaborn as sns
from tenacity import retry, wait_exponential
from tqdm import tqdm
from collections import defaultdict
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import confusion_matrix

import warnings
warnings.filterwarnings('ignore')

tqdm.pandas()

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

```

### Set your keys
Get your OpenAI keys [here](https://platform.openai.com/account/api-keys) and Qdrant keys after making a free cluster [here](https://cloud.qdrant.io/login).


```python
os.environ["QDRANT_URL"] = "https://xxx.cloud.qdrant.io:6333"
os.environ["QDRANT_API_KEY"] = "xxx"
```

## Section A

## 2. Data Preparation: SQuADv2 Data Subsets

For the purpose of demonstration, we'll make small slices from the train and validation splits of the [SQuADv2](https://rajpurkar.github.io/SQuAD-explorer/) dataset. This dataset has questions and contexts where the answer is not present in the context, to help us evaluate how LLM handles this case.

We'll read the data from the JSON files and create a dataframe with the following columns: `question`, `context`, `answer`, `is_impossible`.

### Download the Data


```python
# !mkdir -p local_cache
# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json -O local_cache/train.json
# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O local_cache/dev.json
```

### Read JSON to DataFrame


key: _node_content
string_value:
{"id_": "2741944f-30bd-4890-8462-9b40f7b7d4b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "accd346b-7b95-4fe1-8845-4877c3c0f272", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "de7283f094a9bf414bd7690cdd73c04ea16d54534dbf794ef19e1fff772d508d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "dd7f6a46-dedc-4405-ae40-cf2da70e3c67", "node_type": "1", "metadata": {}, "hash": "a42d924a367cff538724c55c136c21c210e8bfa39cf70b01b6d4e16bc4a83f25", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4062, "end_char_idx": 7384, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
os.environ["QDRANT_URL"] = "https://xxx.cloud.qdrant.io:6333"
os.environ["QDRANT_API_KEY"] = "xxx"
```

## Section A

## 2. Data Preparation: SQuADv2 Data Subsets

For the purpose of demonstration, we'll make small slices from the train and validation splits of the [SQuADv2](https://rajpurkar.github.io/SQuAD-explorer/) dataset. This dataset has questions and contexts where the answer is not present in the context, to help us evaluate how LLM handles this case.

We'll read the data from the JSON files and create a dataframe with the following columns: `question`, `context`, `answer`, `is_impossible`.

### Download the Data


```python
# !mkdir -p local_cache
# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json -O local_cache/train.json
# !wget https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json -O local_cache/dev.json
```

### Read JSON to DataFrame


```python
def json_to_dataframe_with_titles(json_data):
    qas = []
    context = []
    is_impossible = []
    answers = []
    titles = []

    for article in json_data['data']:
        title = article['title']
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                qas.append(qa['question'].strip())
                context.append(paragraph['context'])
                is_impossible.append(qa['is_impossible'])
                
                ans_list = []
                for ans in qa['answers']:
                    ans_list.append(ans['text'])
                answers.append(ans_list)
                titles.append(title)

    df = pd.DataFrame({'title': titles, 'question': qas, 'context': context, 'is_impossible': is_impossible, 'answers': answers})
    return df

def get_diverse_sample(df, sample_size=100, random_state=42):
    """
    Get a diverse sample of the dataframe by sampling from each title
    """
    sample_df = df.groupby(['title', 'is_impossible']).apply(lambda x: x.sample(min(len(x), max(1, sample_size // 50)), random_state=random_state)).reset_index(drop=True)
    
    if len(sample_df) < sample_size:
        remaining_sample_size = sample_size - len(sample_df)
        remaining_df = df.drop(sample_df.index).sample(remaining_sample_size, random_state=random_state)
        sample_df = pd.concat([sample_df, remaining_df]).sample(frac=1, random_state=random_state).reset_index(drop=True)

    return sample_df.sample(min(sample_size, len(sample_df)), random_state=random_state).reset_index(drop=True)

train_df = json_to_dataframe_with_titles(json.load(open('local_cache/train.json')))
val_df = json_to_dataframe_with_titles(json.load(open('local_cache/dev.json')))

df = get_diverse_sample(val_df, sample_size=100, random_state=42)
```

## 3. Answering using Base gpt-3.5-turbo-0613 model

### 3.1 Zero Shot Prompt

Let's start by using the base gpt-3.5-turbo-0613 model to answer the questions. This prompt is a simple concatenation of the question and context, with a separator token in between: `\n\n`. We've a simple instruction part of the prompt: 

> Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.

Other prompts are possible, but this is a good starting point. We'll use this prompt to answer the questions in the validation set.


key: _node_content
string_value:
{"id_": "dd7f6a46-dedc-4405-ae40-cf2da70e3c67", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2741944f-30bd-4890-8462-9b40f7b7d4b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "765fa4aa1ab28f05589ee33dac6f19c268514fcb948909da44f2c9fea4e13bea", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1d592644-26b1-43e3-89c5-e5ab1cb3755e", "node_type": "1", "metadata": {}, "hash": "5ec93f2cc416bf59589b037262979f8372d781ef569a34f7ad36309f7d723244", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Function to get prompt messages
def get_prompt(row):
    return [
        {"role": "system", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": f"""Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.
    Question: {row.question}\n\n
    Context: {row.context}\n\n
    Answer:\n""",
        },
    ]
```

### 3.2 Answering using Zero Shot Prompt

Next, you'll need some re-usable functions which make an OpenAI API Call and return the answer. You'll use the `ChatCompletion.create` endpoint of the API, which takes a prompt and returns the completed text.


```python
# Function with tenacity for retries
@retry(wait=wait_exponential(multiplier=1, min=2, max=6))
def api_call(messages, model):
    return client.chat.completions.create(
        model=model,
        messages=messages,
        stop=["\n\n"],
        max_tokens=100,
        temperature=0.0,
    )


# Main function to answer question
def answer_question(row, prompt_func=get_prompt, model="gpt-3.5-turbo"):
    messages = prompt_func(row)
    response = api_call(messages, model)
    return response.choices[0].message.content
```

⏰ **Time to run: ~3 min**, 🛜 Needs Internet Connection


```python
# Use progress_apply with tqdm for progress bar
df["generated_answer"] = df.progress_apply(answer_question, axis=1)
df.to_json("local_cache/100_val.json", orient="records", lines=True)
df = pd.read_json("local_cache/100_val.json", orient="records", lines=True)
```


```python
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>question</th>
      <th>context</th>
      <th>is_impossible</th>
      <th>answers</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Scottish_Parliament</td>
      <td>What consequence of establishing the Scottish .</td>
      <td>A procedural consequence of the establishment .</td>
      <td>False</td>
      <td>[able to vote on domestic legislation that app.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Imperialism</td>
      <td>Imperialism is less often associated with whic.</td>
      <td>The principles of imperialism are often genera.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Economic_inequality</td>
      <td>What issues can't prevent women from working o.</td>
      <td>When a person’s capabilities are lowered, they.


key: _node_content
string_value:
{"id_": "1d592644-26b1-43e3-89c5-e5ab1cb3755e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "dd7f6a46-dedc-4405-ae40-cf2da70e3c67", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "cc61c7e97b40e65e9fd44e397cdb220ba4f1566393097d81084d00257d0fc808", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "420ded6b-ac72-4303-8df8-5d970fccc332", "node_type": "1", "metadata": {}, "hash": "efbbedffe65f71d470b5caef81673f53f61d570a03f7d2c364ec3ec1a03c3694", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>title</th>
      <th>question</th>
      <th>context</th>
      <th>is_impossible</th>
      <th>answers</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Scottish_Parliament</td>
      <td>What consequence of establishing the Scottish .</td>
      <td>A procedural consequence of the establishment .</td>
      <td>False</td>
      <td>[able to vote on domestic legislation that app.</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Imperialism</td>
      <td>Imperialism is less often associated with whic.</td>
      <td>The principles of imperialism are often genera.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Economic_inequality</td>
      <td>What issues can't prevent women from working o.</td>
      <td>When a person’s capabilities are lowered, they.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Southern_California</td>
      <td>What county are Los Angeles, Orange, San Diego.</td>
      <td>Its counties of Los Angeles, Orange, San Diego.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>4</th>
      <td>French_and_Indian_War</td>
      <td>When was the deportation of Canadians?</td>
      <td>Britain gained control of French Canada and Ac.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>95</th>
      <td>Geology</td>
      <td>In the layered Earth model, what is the inner .</td>
      <td>Seismologists can use the arrival times of sei.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>96</th>
      <td>Prime_number</td>
      <td>What type of value would the Basel function ha.</td>
      <td>The zeta function is closely related to prime .</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>97</th>
      <td>Fresno,


key: _node_content
string_value:
{"id_": "420ded6b-ac72-4303-8df8-5d970fccc332", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1d592644-26b1-43e3-89c5-e5ab1cb3755e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "7831e60d017808890151e9f3db8332b39850fe0fc7498744d2ac6832cbfbf2d5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "333c94dc-8986-433e-aa73-bd22faaec714", "node_type": "1", "metadata": {}, "hash": "5e4794ada91c8216cc43a36727992fa6a9e01640642155da2528350b4fd6f811", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
they.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Southern_California</td>
      <td>What county are Los Angeles, Orange, San Diego.</td>
      <td>Its counties of Los Angeles, Orange, San Diego.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>4</th>
      <td>French_and_Indian_War</td>
      <td>When was the deportation of Canadians?</td>
      <td>Britain gained control of French Canada and Ac.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>.</th>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
      <td>.</td>
    </tr>
    <tr>
      <th>95</th>
      <td>Geology</td>
      <td>In the layered Earth model, what is the inner .</td>
      <td>Seismologists can use the arrival times of sei.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>96</th>
      <td>Prime_number</td>
      <td>What type of value would the Basel function ha.</td>
      <td>The zeta function is closely related to prime .</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>97</th>
      <td>Fresno,_California</td>
      <td>What does the San Joaquin Valley Railroad cros.</td>
      <td>Passenger rail service is provided by Amtrak S.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>98</th>
      <td>Victoria_(Australia)</td>
      <td>What party rules in Melbourne's inner regions?</td>
      <td>The centre-left Australian Labor Party (ALP), .</td>
      <td>False</td>
      <td>[The Greens, Australian Greens, Greens]</td>
    </tr>
    <tr>
      <th>99</th>
      <td>Immune_system</td>
      <td>The speed of the killing response of the human.</td>
      <td>In humans, this response is activated by compl.</td>
      <td>False</td>
      <td>[signal amplification, signal amplification, s.</td>
    </tr>
  </tbody>
</table>
<p>100 rows × 5 columns</p>
</div>


key: _node_content
string_value:
{"id_": "333c94dc-8986-433e-aa73-bd22faaec714", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "420ded6b-ac72-4303-8df8-5d970fccc332", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "f27232efe20d552226e4ac24ad7241b125f8ae209afec0ddb844d2fef34c2cea", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "528744c6-dc74-44d3-844f-08e37a8d6785", "node_type": "1", "metadata": {}, "hash": "57ff9005f07cbfaec771da9f4ba06230b120d978ef00ec573a97af03aa38c9f8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
what is the inner .</td>
      <td>Seismologists can use the arrival times of sei.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>96</th>
      <td>Prime_number</td>
      <td>What type of value would the Basel function ha.</td>
      <td>The zeta function is closely related to prime .</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>97</th>
      <td>Fresno,_California</td>
      <td>What does the San Joaquin Valley Railroad cros.</td>
      <td>Passenger rail service is provided by Amtrak S.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>98</th>
      <td>Victoria_(Australia)</td>
      <td>What party rules in Melbourne's inner regions?</td>
      <td>The centre-left Australian Labor Party (ALP), .</td>
      <td>False</td>
      <td>[The Greens, Australian Greens, Greens]</td>
    </tr>
    <tr>
      <th>99</th>
      <td>Immune_system</td>
      <td>The speed of the killing response of the human.</td>
      <td>In humans, this response is activated by compl.</td>
      <td>False</td>
      <td>[signal amplification, signal amplification, s.</td>
    </tr>
  </tbody>
</table>
<p>100 rows × 5 columns</p>
</div>



## 4. Fine-tuning and Answering using Fine-tuned model

For the complete fine-tuning process, please refer to the [OpenAI Fine-Tuning Docs](https://platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model).

### 4.1 Prepare the Fine-Tuning Data

We need to prepare the data for fine-tuning. We'll use a few samples from train split of same dataset as before, but we'll add the answer to the context. This will help the model learn to retrieve the answer from the context. 

Our instruction prompt is the same as before, and so is the system prompt.


key: _node_content
string_value:
{"id_": "528744c6-dc74-44d3-844f-08e37a8d6785", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "333c94dc-8986-433e-aa73-bd22faaec714", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "0149784cb927f00e9c54f3ef1511595e6dbbb7f74d378c00d532093881ae7a58", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bfc88736-235b-4a11-9ed1-18509240fde2", "node_type": "1", "metadata": {}, "hash": "6f74cef954d61cb79d144ad5fdf416e110044fb0132d3092b591f38daf10636c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
_California</td>
      <td>What does the San Joaquin Valley Railroad cros.</td>
      <td>Passenger rail service is provided by Amtrak S.</td>
      <td>True</td>
      <td>[]</td>
    </tr>
    <tr>
      <th>98</th>
      <td>Victoria_(Australia)</td>
      <td>What party rules in Melbourne's inner regions?</td>
      <td>The centre-left Australian Labor Party (ALP), .</td>
      <td>False</td>
      <td>[The Greens, Australian Greens, Greens]</td>
    </tr>
    <tr>
      <th>99</th>
      <td>Immune_system</td>
      <td>The speed of the killing response of the human.</td>
      <td>In humans, this response is activated by compl.</td>
      <td>False</td>
      <td>[signal amplification, signal amplification, s.</td>
    </tr>
  </tbody>
</table>
<p>100 rows × 5 columns</p>
</div>



## 4. Fine-tuning and Answering using Fine-tuned model

For the complete fine-tuning process, please refer to the [OpenAI Fine-Tuning Docs](https://platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model).

### 4.1 Prepare the Fine-Tuning Data

We need to prepare the data for fine-tuning. We'll use a few samples from train split of same dataset as before, but we'll add the answer to the context. This will help the model learn to retrieve the answer from the context. 

Our instruction prompt is the same as before, and so is the system prompt. 


```python
def dataframe_to_jsonl(df):
    def create_jsonl_entry(row):
        answer = row["answers"][0] if row["answers"] else "I don't know"
        messages = [
            {"role": "system", "content": "You are a helpful assistant."},
            {
                "role": "user",
                "content": f"""Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.
            Question: {row.question}\n\n
            Context: {row.context}\n\n
            Answer:\n""",
            },
            {"role": "assistant", "content": answer},
        ]
        return json.dumps({"messages": messages})

    jsonl_output = df.apply(create_jsonl_entry, axis=1)
    return "\n".join(jsonl_output)

train_sample = get_diverse_sample(train_df, sample_size=100, random_state=42)

with open("local_cache/100_train.jsonl", "w") as f:
    f.write(dataframe_to_jsonl(train_sample))
```

**Tip: 💡 Verify the Fine-Tuning Data**

You can see this [cookbook](https://github.com/openai/openai-cookbook/blob/main/examples/Chat_finetuning_data_prep.ipynb) for more details on how to prepare the data for fine-tuning.

### 4.2 Fine-Tune OpenAI Model

If you're new to OpenAI Model Fine-Tuning, please refer to the [How to finetune Chat models](https://github.com/openai/openai-cookbook/blob/448a0595b84ced3bebc9a1568b625e748f9c1d60/examples/How_to_finetune_chat_models.ipynb) notebook. You can also refer to the [OpenAI Fine-Tuning Docs](platform.openai.com/docs/guides/fine-tuning/use-a-fine-tuned-model) for more details.


key: _node_content
string_value:
{"id_": "bfc88736-235b-4a11-9ed1-18509240fde2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "528744c6-dc74-44d3-844f-08e37a8d6785", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "7095abf517940c201bc7be50b51e3d08c9bf4a16ce96e616819aa410215d667b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "acf54560-db4e-41a1-a323-36d8366de716", "node_type": "1", "metadata": {}, "hash": "0578ada283197b1b1d56ba723adb382013efac4a40dee49e006c32e9c5db664e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14308, "end_char_idx": 17379, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
class OpenAIFineTuner:
    """
    Class to fine tune OpenAI models
    """
    def __init__(self, training_file_path, model_name, suffix):
        self.training_file_path = training_file_path
        self.model_name = model_name
        self.suffix = suffix
        self.file_object = None
        self.fine_tuning_job = None
        self.model_id = None

    def create_openai_file(self):
        self.file_object = client.files.create(
            file=open(self.training_file_path, "r"),
            purpose="fine-tune",
        )

    def wait_for_file_processing(self, sleep_time=20):
        while self.file_object.status != 'processed':
            time.sleep(sleep_time)
            self.file_object.refresh()
            print("File Status: ", self.file_object.status)

    def create_fine_tuning_job(self):
        self.fine_tuning_job = client.fine_tuning.jobs.create(
            training_file=self.file_object["id"],
            model=self.model_name,
            suffix=self.suffix,
        )

    def wait_for_fine_tuning(self, sleep_time=45):
        while self.fine_tuning_job.status != 'succeeded':
            time.sleep(sleep_time)
            self.fine_tuning_job.refresh()
            print("Job Status: ", self.fine_tuning_job.status)

    def retrieve_fine_tuned_model(self):
        self.model_id = client.fine_tuning.jobs.retrieve(self.fine_tuning_job["id"]).fine_tuned_model
        return self.model_id

    def fine_tune_model(self):
        self.create_openai_file()
        self.wait_for_file_processing()
        self.create_fine_tuning_job()
        self.wait_for_fine_tuning()
        return self.retrieve_fine_tuned_model()

fine_tuner = OpenAIFineTuner(
        training_file_path="local_cache/100_train.jsonl",
        model_name="gpt-3.5-turbo",
        suffix="100trn20230907"
    )
```

⏰ **Time to run: ~10-20 minutes**, 🛜 Needs Internet Connection


```python
model_id = fine_tuner.fine_tune_model()
model_id
```

#### 4.2.1 Try out the Fine-Tuned Model

Let's try out the fine-tuned model on the same validation set as before. You'll use the same prompt as before, but you will use the fine-tuned model instead of the base model. Before you do that, you can make a simple call to get a sense of how the fine-tuned model is doing.


```python
completion = client.chat.completions.create(
    model=model_id,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Hello!"},
        {"role": "assistant", "content": "Hi, how can I help you today?"},
        {
            "role": "user",
            "content": "Can you answer the following question based on the given context? If not, say, I don't know:\n\nQuestion: What is the capital of France?\n\nContext: The capital of Mars is Gaia. Answer:",
        },
    ],
)

print(completion.choices[0].message)
```

### 4.3 Answer Using the Fine-Tuned Model

This is the same as before, but you'll use the fine-tuned model instead of the base model.

⏰ **Time to run: ~5 min**, 🛜 Needs Internet Connection


key: _node_content
string_value:
{"id_": "acf54560-db4e-41a1-a323-36d8366de716", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "bfc88736-235b-4a11-9ed1-18509240fde2", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6a2d9f32a06e425ac72ac1b1126ef0d0ff325a2bb069784b19e0acdfda1b15f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2b5256f9-d781-4d7d-a832-dce5d04929ea", "node_type": "1", "metadata": {}, "hash": "72fc6e7c8ac5cfa9916d6ea58a4fa655ae496f8e23a6b2f0de42a9ec376ba68a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16211, "end_char_idx": 19255, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
model_id = fine_tuner.fine_tune_model()
model_id
```

#### 4.2.1 Try out the Fine-Tuned Model

Let's try out the fine-tuned model on the same validation set as before. You'll use the same prompt as before, but you will use the fine-tuned model instead of the base model. Before you do that, you can make a simple call to get a sense of how the fine-tuned model is doing.


```python
completion = client.chat.completions.create(
    model=model_id,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Hello!"},
        {"role": "assistant", "content": "Hi, how can I help you today?"},
        {
            "role": "user",
            "content": "Can you answer the following question based on the given context? If not, say, I don't know:\n\nQuestion: What is the capital of France?\n\nContext: The capital of Mars is Gaia. Answer:",
        },
    ],
)

print(completion.choices[0].message)
```

### 4.3 Answer Using the Fine-Tuned Model

This is the same as before, but you'll use the fine-tuned model instead of the base model.

⏰ **Time to run: ~5 min**, 🛜 Needs Internet Connection


```python
df["ft_generated_answer"] = df.progress_apply(answer_question, model=model_id, axis=1)
```

## 5. Evaluation: How well does the model perform?

To evaluate the model's performance, compare the predicted answer to the actual answers -- if any of the actual answers are present in the predicted answer, then it's a match. We've also created error categories to help you understand where the model is struggling.

When we know that a correct answer exists in the context, we can measure the model's performance, there are 3 possible outcomes:

1. ✅ **Answered Correctly**: The model responded the correct answer. It may have also included other answers that were not in the context.
2. ❎ **Skipped**: The model responded with "I don't know" (IDK) while the answer was present in the context. It's better than giving the wrong answer. It's better for the model say "I don't know" than giving the wrong answer. In our design, we know that a true answer exists and hence we're able to measure it -- this is not always the case. *This is a model error*. We exclude this from the overall error rate. 
3. ❌ **Wrong**: The model responded with an incorrect answer. **This is a model ERROR.**

When we know that a correct answer does not exist in the context, we can measure the model's performance, there are 2 possible outcomes:

4. ❌ **Hallucination**: The model responded with an answer, when "I don't know" was expected. **This is a model ERROR.** 
5. ✅ **I don't know**: The model responded with "I don't know" (IDK) and the answer was not present in the context. **This is a model WIN.**


```python
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

class Evaluator:
    def __init__(self, df):
        self.df = df
        self.y_pred = pd.Series()  # Initialize as empty Series
        self.labels_answer_expected = ["✅ Answered Correctly",


key: _node_content
string_value:
{"id_": "2b5256f9-d781-4d7d-a832-dce5d04929ea", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "acf54560-db4e-41a1-a323-36d8366de716", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a6de42308c7df856bd34ee95453ff5623c8dcf92a6cbf06434114ceba1ff9673", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e3153ba1-be3b-4859-ade3-5b12d2ee5db0", "node_type": "1", "metadata": {}, "hash": "5858c6253cd9e7d175c88a6911cf65be84d65f6fcac5b570b495ea27d5656bc2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 18978, "end_char_idx": 22144, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

class Evaluator:
    def __init__(self, df):
        self.df = df
        self.y_pred = pd.Series()  # Initialize as empty Series
        self.labels_answer_expected = ["✅ Answered Correctly", "❎ Skipped", "❌ Wrong Answer"]
        self.labels_idk_expected = ["❌ Hallucination", "✅ I don't know"]

    def _evaluate_answer_expected(self, row, answers_column):
        generated_answer = row[answers_column].lower()
        actual_answers = [ans.lower() for ans in row["answers"]]
        return (
            "✅ Answered Correctly" if any(ans in generated_answer for ans in actual_answers)
            else "❎ Skipped" if generated_answer == "i don't know"
            else "❌ Wrong Answer"
        )

    def _evaluate_idk_expected(self, row, answers_column):
        generated_answer = row[answers_column].lower()
        return (
            "❌ Hallucination" if generated_answer != "i don't know"
            else "✅ I don't know"
        )

    def _evaluate_single_row(self, row, answers_column):
        is_impossible = row["is_impossible"]
        return (
            self._evaluate_answer_expected(row, answers_column) if not is_impossible
            else self._evaluate_idk_expected(row, answers_column)
        )

    def evaluate_model(self, answers_column="generated_answer"):
        self.y_pred = pd.Series(self.df.apply(self._evaluate_single_row, answers_column=answers_column, axis=1))
        freq_series = self.y_pred.value_counts()
        
        # Counting rows for each scenario
        total_answer_expected = len(self.df[self.df['is_impossible'] == False])
        total_idk_expected = len(self.df[self.df['is_impossible'] == True])
        
        freq_answer_expected = (freq_series / total_answer_expected * 100).round(2).reindex(self.labels_answer_expected, fill_value=0)
        freq_idk_expected = (freq_series / total_idk_expected * 100).round(2).reindex(self.labels_idk_expected, fill_value=0)
        return freq_answer_expected.to_dict(), freq_idk_expected.to_dict()

    def print_eval(self):
        answer_columns=["generated_answer", "ft_generated_answer"]
        baseline_correctness, baseline_idk = self.evaluate_model()
        ft_correctness, ft_idk = self.evaluate_model(self.df, answer_columns[1])
        print("When the model should answer correctly:")
        eval_df = pd.merge(
            baseline_correctness.rename("Baseline"),
            ft_correctness.rename("Fine-Tuned"),
            left_index=True,
            right_index=True,
        )
        print(eval_df)
        print("\n\n\nWhen the model should say 'I don't know':")
        eval_df = pd.merge(
            baseline_idk.rename("Baseline"),
            ft_idk.rename("Fine-Tuned"),
            left_index=True,
            right_index=True,
        )
        print(eval_df)
    
    def plot_model_comparison(self, answer_columns=["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"]):
        
        results = []
        for col in answer_columns:
            answer_expected, idk_expected = self.


key: _node_content
string_value:
{"id_": "e3153ba1-be3b-4859-ade3-5b12d2ee5db0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2b5256f9-d781-4d7d-a832-dce5d04929ea", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "d3bc307da9e6f30994b9982b1b93341a1e9cdf00dea3c30a1d51ddab2d61485d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "49167a19-abba-4ecb-b91d-20417ff6c96c", "node_type": "1", "metadata": {}, "hash": "9368a004517e47a9c6db873231720c30d5af3c13df7ead400b6db56a8d56920c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 20428, "end_char_idx": 23521, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
answers_column=answers_column, axis=1))
        freq_series = self.y_pred.value_counts()
        
        # Counting rows for each scenario
        total_answer_expected = len(self.df[self.df['is_impossible'] == False])
        total_idk_expected = len(self.df[self.df['is_impossible'] == True])
        
        freq_answer_expected = (freq_series / total_answer_expected * 100).round(2).reindex(self.labels_answer_expected, fill_value=0)
        freq_idk_expected = (freq_series / total_idk_expected * 100).round(2).reindex(self.labels_idk_expected, fill_value=0)
        return freq_answer_expected.to_dict(), freq_idk_expected.to_dict()

    def print_eval(self):
        answer_columns=["generated_answer", "ft_generated_answer"]
        baseline_correctness, baseline_idk = self.evaluate_model()
        ft_correctness, ft_idk = self.evaluate_model(self.df, answer_columns[1])
        print("When the model should answer correctly:")
        eval_df = pd.merge(
            baseline_correctness.rename("Baseline"),
            ft_correctness.rename("Fine-Tuned"),
            left_index=True,
            right_index=True,
        )
        print(eval_df)
        print("\n\n\nWhen the model should say 'I don't know':")
        eval_df = pd.merge(
            baseline_idk.rename("Baseline"),
            ft_idk.rename("Fine-Tuned"),
            left_index=True,
            right_index=True,
        )
        print(eval_df)
    
    def plot_model_comparison(self, answer_columns=["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"]):
        
        results = []
        for col in answer_columns:
            answer_expected, idk_expected = self.evaluate_model(col)
            if scenario == "answer_expected":
                results.append(answer_expected)
            elif scenario == "idk_expected":
                results.append(idk_expected)
            else:
                raise ValueError("Invalid scenario")
        
        
        results_df = pd.DataFrame(results, index=nice_names)
        if scenario == "answer_expected":
            results_df = results_df.reindex(self.labels_answer_expected, axis=1)
        elif scenario == "idk_expected":
            results_df = results_df.reindex(self.labels_idk_expected, axis=1)
        
        melted_df = results_df.reset_index().melt(id_vars='index', var_name='Status', value_name='Frequency')
        sns.set_theme(style="whitegrid", palette="icefire")
        g = sns.catplot(data=melted_df, x='Frequency', y='index', hue='Status', kind='bar', height=5, aspect=2)

        # Annotating each bar
        for p in g.ax.patches:
            g.ax.annotate(f"{p.get_width():.0f}%", (p.get_width()+5, p.get_y() + p.get_height() / 2),
                        textcoords="offset points",
                        xytext=(0, 0),
                        ha='center', va='center')
        plt.ylabel("Model")
        plt.xlabel("Percentage")
        plt.xlim(0, 100)
        plt.tight_layout()
        plt.title(scenario.replace("_", " ").title())
        plt.show()


key: _node_content
string_value:
{"id_": "49167a19-abba-4ecb-b91d-20417ff6c96c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e3153ba1-be3b-4859-ade3-5b12d2ee5db0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "21d0feea35e5e6c0d36eccea2298ca6ad68d66ccfbe56bef1571f8b03d40955e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fd215b19-36c2-4ca3-8f71-03ca259297e6", "node_type": "1", "metadata": {}, "hash": "663f178ba2e260e7db4643b5e6bb0297e1060b07d7c8b3c1fef73f98059f028a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 21836, "end_char_idx": 24602, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
)
        print(eval_df)
    
    def plot_model_comparison(self, answer_columns=["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"]):
        
        results = []
        for col in answer_columns:
            answer_expected, idk_expected = self.evaluate_model(col)
            if scenario == "answer_expected":
                results.append(answer_expected)
            elif scenario == "idk_expected":
                results.append(idk_expected)
            else:
                raise ValueError("Invalid scenario")
        
        
        results_df = pd.DataFrame(results, index=nice_names)
        if scenario == "answer_expected":
            results_df = results_df.reindex(self.labels_answer_expected, axis=1)
        elif scenario == "idk_expected":
            results_df = results_df.reindex(self.labels_idk_expected, axis=1)
        
        melted_df = results_df.reset_index().melt(id_vars='index', var_name='Status', value_name='Frequency')
        sns.set_theme(style="whitegrid", palette="icefire")
        g = sns.catplot(data=melted_df, x='Frequency', y='index', hue='Status', kind='bar', height=5, aspect=2)

        # Annotating each bar
        for p in g.ax.patches:
            g.ax.annotate(f"{p.get_width():.0f}%", (p.get_width()+5, p.get_y() + p.get_height() / 2),
                        textcoords="offset points",
                        xytext=(0, 0),
                        ha='center', va='center')
        plt.ylabel("Model")
        plt.xlabel("Percentage")
        plt.xlim(0, 100)
        plt.tight_layout()
        plt.title(scenario.replace("_", " ").title())
        plt.show()


# Compare the results by merging into one dataframe
evaluator = Evaluator(df)
# evaluator.evaluate_model(answers_column="ft_generated_answer")
# evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"])
```


```python
# Optionally, save the results to a JSON file
df.to_json("local_cache/100_val_ft.json", orient="records", lines=True)
df = pd.read_json("local_cache/100_val_ft.json", orient="records", lines=True)
```


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"])
```


    
![png](output_31_0.png)
    


Notice that the fine-tuned model skips questions more often -- and makes fewer mistakes. This is because the fine-tuned model is more conservative and skips questions when it's not sure.


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="idk_expected", nice_names=["Baseline", "Fine-Tuned"])
```


    
![png](output_33_0.png)


key: _node_content
string_value:
{"id_": "fd215b19-36c2-4ca3-8f71-03ca259297e6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "49167a19-abba-4ecb-b91d-20417ff6c96c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "3dcbb060eda08f187960285837b0473be5b5407eb8d4641456a581ae350fb26d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6aacc9d4-f575-4db8-862a-5b399c9c5c64", "node_type": "1", "metadata": {}, "hash": "c61f4fbdbc155314b5d0a1841f69a992b85edfca1e170edba8d2c6d1681c49fc", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 22999, "end_char_idx": 25988, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
kind='bar', height=5, aspect=2)

        # Annotating each bar
        for p in g.ax.patches:
            g.ax.annotate(f"{p.get_width():.0f}%", (p.get_width()+5, p.get_y() + p.get_height() / 2),
                        textcoords="offset points",
                        xytext=(0, 0),
                        ha='center', va='center')
        plt.ylabel("Model")
        plt.xlabel("Percentage")
        plt.xlim(0, 100)
        plt.tight_layout()
        plt.title(scenario.replace("_", " ").title())
        plt.show()


# Compare the results by merging into one dataframe
evaluator = Evaluator(df)
# evaluator.evaluate_model(answers_column="ft_generated_answer")
# evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"])
```


```python
# Optionally, save the results to a JSON file
df.to_json("local_cache/100_val_ft.json", orient="records", lines=True)
df = pd.read_json("local_cache/100_val_ft.json", orient="records", lines=True)
```


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned"])
```


    
![png](output_31_0.png)
    


Notice that the fine-tuned model skips questions more often -- and makes fewer mistakes. This is because the fine-tuned model is more conservative and skips questions when it's not sure.


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer"], scenario="idk_expected", nice_names=["Baseline", "Fine-Tuned"])
```


    
![png](output_33_0.png)
    


Notice that the fine-tuned model has learnt to say "I don't know" a lot better than the prompt. Or, the model has gotten good at skipping questions.

### Observations

1. The fine-tuned model is better at saying "I don't know"
2. Hallucinations drop from 100% to 15% with fine-tuning
3. Wrong answers drop from 17% to 6% with fine-tuning

**Correct answers also drop from 83% to 60% with fine-tuning** - this is because the fine-tuned model is **more conservative** and says "I don't know" more often. This is a good thing because it's better to say "I don't know" than to give a wrong answer.

That said, we want to improve the correctness of the model, even if that increases the hallucinations. We're looking for a model that is both correct and conservative, striking a balance between the two. We'll use Qdrant and Few-Shot Learning to achieve this.

**💪 You're 2/3rds of the way there! Keep reading!**

# Section B: Few Shot Learning

We'll select a few examples from the dataset, including cases where the answer is not present in the context. We'll then use these examples to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model.

**What is next?**

6. Fine-Tuning OpenAI Model with Qdrant
    6.1 Embed the Fine-Tuning Data
    6.2 Embedding the Questions
7. Using Qdrant to Improve RAG Prompt
8. Evaluation


key: _node_content
string_value:
{"id_": "6aacc9d4-f575-4db8-862a-5b399c9c5c64", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fd215b19-36c2-4ca3-8f71-03ca259297e6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "75bd175cce004d756277e499920c3f9809d35a1c684b8d803d0d00f7f5be3f92", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "abcddb3b-f86c-408e-9f91-4538328ca64f", "node_type": "1", "metadata": {}, "hash": "5e5e9556d987a7242052fe0565395d8bf34e4cc5b8a3ae8b6b16e66bc0c9fec8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 24579, "end_char_idx": 27505, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
![png](output_33_0.png)
    


Notice that the fine-tuned model has learnt to say "I don't know" a lot better than the prompt. Or, the model has gotten good at skipping questions.

### Observations

1. The fine-tuned model is better at saying "I don't know"
2. Hallucinations drop from 100% to 15% with fine-tuning
3. Wrong answers drop from 17% to 6% with fine-tuning

**Correct answers also drop from 83% to 60% with fine-tuning** - this is because the fine-tuned model is **more conservative** and says "I don't know" more often. This is a good thing because it's better to say "I don't know" than to give a wrong answer.

That said, we want to improve the correctness of the model, even if that increases the hallucinations. We're looking for a model that is both correct and conservative, striking a balance between the two. We'll use Qdrant and Few-Shot Learning to achieve this.

**💪 You're 2/3rds of the way there! Keep reading!**

# Section B: Few Shot Learning

We'll select a few examples from the dataset, including cases where the answer is not present in the context. We'll then use these examples to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model.

**What is next?**

6. Fine-Tuning OpenAI Model with Qdrant
    6.1 Embed the Fine-Tuning Data
    6.2 Embedding the Questions
7. Using Qdrant to Improve RAG Prompt
8. Evaluation


## 6. Fine-Tuning OpenAI Model with Qdrant

So far, we've been using the OpenAI model to answer questions without using examples of the answer. The previous step made it work better on in-context examples, while this one helps it generalize to unseen data, and attempt to learn when to say "I don't know" and when to give an answer.

This is where few-shot learning comes in!

Few-shot learning is a type of transfer learning that allows us to answer questions where the answer is not present in the context. We can do this by providing a few examples of the answer we're looking for, and the model will learn to answer questions where the answer is not present in the context.

### 5.1 Embed the Training Data

Embeddings are a way to represent sentences as an array of floats. We'll use the embeddings to find the most similar questions to the ones we're looking for.


```python
import os
from qdrant_client import QdrantClient
from qdrant_client.http import models
from qdrant_client.http.models import PointStruct
from qdrant_client.http.models import Distance, VectorParams
```

Now that we've the Qdrant imports in place, 


```python
qdrant_client = QdrantClient(
    url=os.getenv("QDRANT_URL"), api_key=os.getenv("QDRANT_API_KEY"), timeout=6000, prefer_grpc=True
)

collection_name = "squadv2-cookbook"

# # Create the collection, run this only once
# qdrant_client.recreate_collection(
#     collection_name=collection_name,
#     vectors_config=VectorParams(size=384, distance=Distance.COSINE),
# )
```


key: _node_content
string_value:
{"id_": "abcddb3b-f86c-408e-9f91-4538328ca64f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6aacc9d4-f575-4db8-862a-5b399c9c5c64", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "29ea85cec66274dcdac30affb7088360b012840b19a63b4f1612f854bf38a18d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "75159baf-e7e6-428e-aac4-3ffdeca4c173", "node_type": "1", "metadata": {}, "hash": "4ddbe040edf04031d558295a41f48717105f741abfdcfc4d4e680d46f6d4023a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 25991, "end_char_idx": 29363, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 6. Fine-Tuning OpenAI Model with Qdrant

So far, we've been using the OpenAI model to answer questions without using examples of the answer. The previous step made it work better on in-context examples, while this one helps it generalize to unseen data, and attempt to learn when to say "I don't know" and when to give an answer.

This is where few-shot learning comes in!

Few-shot learning is a type of transfer learning that allows us to answer questions where the answer is not present in the context. We can do this by providing a few examples of the answer we're looking for, and the model will learn to answer questions where the answer is not present in the context.

### 5.1 Embed the Training Data

Embeddings are a way to represent sentences as an array of floats. We'll use the embeddings to find the most similar questions to the ones we're looking for.


```python
import os
from qdrant_client import QdrantClient
from qdrant_client.http import models
from qdrant_client.http.models import PointStruct
from qdrant_client.http.models import Distance, VectorParams
```

Now that we've the Qdrant imports in place, 


```python
qdrant_client = QdrantClient(
    url=os.getenv("QDRANT_URL"), api_key=os.getenv("QDRANT_API_KEY"), timeout=6000, prefer_grpc=True
)

collection_name = "squadv2-cookbook"

# # Create the collection, run this only once
# qdrant_client.recreate_collection(
#     collection_name=collection_name,
#     vectors_config=VectorParams(size=384, distance=Distance.COSINE),
# )
```


```python
from fastembed.embedding import DefaultEmbedding
from typing import List
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm

tqdm.pandas()

embedding_model = DefaultEmbedding()
```

### 5.2 Embedding the Questions

Next, you'll embed the entire training set questions. You'll use the question to question similarity to find the most similar questions to the question we're looking for. This is a workflow which is used in RAG to leverage the OpenAI model ability of incontext learning with more examples. This is what we call Few Shot Learning here.

**❗️⏰ Important Note: This step can take up to 3 hours to complete. Please be patient. If you see Out of Memory errors or Kernel Crashes, please reduce the batch size to 32, restart the kernel and run the notebook again. This code needs to be run only ONCE.**

## Function Breakdown for `generate_points_from_dataframe`

1. **Initialization**: `batch_size = 512` and `total_batches` set the stage for how many questions will be processed in one go. This is to prevent memory issues. If your machine can handle more, feel free to increase the batch size. If your kernel crashes, reduce the batch size to 32 and try again.
2. **Progress Bar**: `tqdm` gives you a nice progress bar so you don't fall asleep.
3. **Batch Loop**: The for-loop iterates through batches. `start_idx` and `end_idx` define the slice of the DataFrame to process.
4. **Generate Embeddings**: `batch_embeddings = embedding_model.embed(batch, batch_size=batch_size)` - This is where the magic happens. Your questions get turned into embeddings.
5. **PointStruct Generation**: Using `.progress_apply`, it turns each row into a `PointStruct` object. This includes an ID, the embedding vector, and other metadata.

Returns the list of `PointStruct` objects, which can be used to create a collection in Qdrant.


key: _node_content
string_value:
{"id_": "75159baf-e7e6-428e-aac4-3ffdeca4c173", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "abcddb3b-f86c-408e-9f91-4538328ca64f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "a565d02a5da5b3244d9f94a22859aa73383459d0b613e7c9a786c171b92218e9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b3118562-c63c-4938-86da-3b2e94b04e6b", "node_type": "1", "metadata": {}, "hash": "34b05026f1128ad6ed6c5af64c81e0da5574c844af38eb224a71b3a29c6b1676", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 29366, "end_char_idx": 32460, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def generate_points_from_dataframe(df: pd.DataFrame) -> List[PointStruct]:
    batch_size = 512
    questions = df["question"].tolist()
    total_batches = len(questions) // batch_size + 1
    
    pbar = tqdm(total=len(questions), desc="Generating embeddings")
    
    # Generate embeddings in batches to improve performance
    embeddings = []
    for i in range(total_batches):
        start_idx = i * batch_size
        end_idx = min((i + 1) * batch_size, len(questions))
        batch = questions[start_idx:end_idx]
        
        batch_embeddings = embedding_model.embed(batch, batch_size=batch_size)
        embeddings.extend(batch_embeddings)
        pbar.update(len(batch))
        
    pbar.close()
    
    # Convert embeddings to list of lists
    embeddings_list = [embedding.tolist() for embedding in embeddings]
    
    # Create a temporary DataFrame to hold the embeddings and existing DataFrame columns
    temp_df = df.copy()
    temp_df["embeddings"] = embeddings_list
    temp_df["id"] = temp_df.index
    
    # Generate PointStruct objects using DataFrame apply method
    points = temp_df.progress_apply(
        lambda row: PointStruct(
            id=row["id"],
            vector=row["embeddings"],
            payload={
                "question": row["question"],
                "title": row["title"],
                "context": row["context"],
                "is_impossible": row["is_impossible"],
                "answers": row["answers"],
            },
        ),
        axis=1,
    ).tolist()

    return points

points = generate_points_from_dataframe(train_df)
```

#### Upload the Embeddings to Qdrant

Note that configuring Qdrant is outside the scope of this notebook. Please refer to the [Qdrant](https://qdrant.tech) for more information. We used a timeout of 600 seconds for the upload, and grpc compression to speed up the upload.


```python
operation_info = qdrant_client.upsert(
    collection_name=collection_name, wait=True, points=points
)
print(operation_info)
```

## 6. Using Qdrant to Improve RAG Prompt

Now that we've uploaded the embeddings to Qdrant, we can use Qdrant to find the most similar questions to the question we're looking for. We'll use the top 5 most similar questions to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model on the same validation set, but with few shot prompting!

Our main function `get_few_shot_prompt` serves as the workhorse for generating prompts for few-shot learning. It does this by retrieving similar questions from Qdrant - a vector search engine, using an embeddings model. Here is the high-level workflow:

1. Retrieve similar questions from Qdrant where the **answer is present** in the context
2. Retrieve similar questions from Qdrant where the **answer is IMPOSSIBLE** i.e. the expected answer is "I don't know" to find in the context
3. Create a prompt using the retrieved questions
4. Fine-tune the model using the prompt
5. Evaluate the fine-tuned model on the validation set with the same prompting technique


key: _node_content
string_value:
{"id_": "b3118562-c63c-4938-86da-3b2e94b04e6b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "75159baf-e7e6-428e-aac4-3ffdeca4c173", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "04d1176066f63efda25c8f9ef47c5355cda44a78ff94ebf8cd5cfcce0c16b5bc", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "cb51689e-c778-48a8-b989-4d0d64089391", "node_type": "1", "metadata": {}, "hash": "5e3491d2322ca3cdb399f4440c534a9658fef59a6dacb14386e4de1caf7bf43f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 31258, "end_char_idx": 33752, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
operation_info = qdrant_client.upsert(
    collection_name=collection_name, wait=True, points=points
)
print(operation_info)
```

## 6. Using Qdrant to Improve RAG Prompt

Now that we've uploaded the embeddings to Qdrant, we can use Qdrant to find the most similar questions to the question we're looking for. We'll use the top 5 most similar questions to create a prompt that we can use to fine-tune the model. We'll then measure the performance of the fine-tuned model on the same validation set, but with few shot prompting!

Our main function `get_few_shot_prompt` serves as the workhorse for generating prompts for few-shot learning. It does this by retrieving similar questions from Qdrant - a vector search engine, using an embeddings model. Here is the high-level workflow:

1. Retrieve similar questions from Qdrant where the **answer is present** in the context
2. Retrieve similar questions from Qdrant where the **answer is IMPOSSIBLE** i.e. the expected answer is "I don't know" to find in the context
3. Create a prompt using the retrieved questions
4. Fine-tune the model using the prompt
5. Evaluate the fine-tuned model on the validation set with the same prompting technique


```python
def get_few_shot_prompt(row):

    query, row_context = row["question"], row["context"]

    embeddings = list(embedding_model.embed([query]))
    query_embedding = embeddings[0].tolist()

    num_of_qa_to_retrieve = 5

    # Query Qdrant for similar questions that have an answer
    q1 = qdrant_client.search(
        collection_name=collection_name,
        query_vector=query_embedding,
        with_payload=True,
        limit=num_of_qa_to_retrieve,
        query_filter=models.Filter(
            must=[
                models.FieldCondition(
                    key="is_impossible",
                    match=models.MatchValue(
                        value=False,
                    ),
                ),
            ],
        )
    )

    # Query Qdrant for similar questions that are IMPOSSIBLE to answer
    q2 = qdrant_client.search(
        collection_name=collection_name,
        query_vector=query_embedding,
        query_filter=models.Filter(
            must=[
                models.FieldCondition(
                    key="is_impossible",
                    match=models.MatchValue(
                        value=True,
                    ),
                ),
            ]
        ),
        with_payload=True,
        limit=num_of_qa_to_retrieve,
    )


key: _node_content
string_value:
{"id_": "cb51689e-c778-48a8-b989-4d0d64089391", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b3118562-c63c-4938-86da-3b2e94b04e6b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "94d5ef521d124cf05b4c8c0d7dd8dfb563f317f122055c0568bf81b6ba6cd493", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "beee9ec9-e146-4400-9276-d42a23938c9c", "node_type": "1", "metadata": {}, "hash": "29cd50c31aaeff745085d82438d98308905ae83f27af17558d6287056d9c8211", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 32463, "end_char_idx": 35609, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def get_few_shot_prompt(row):

    query, row_context = row["question"], row["context"]

    embeddings = list(embedding_model.embed([query]))
    query_embedding = embeddings[0].tolist()

    num_of_qa_to_retrieve = 5

    # Query Qdrant for similar questions that have an answer
    q1 = qdrant_client.search(
        collection_name=collection_name,
        query_vector=query_embedding,
        with_payload=True,
        limit=num_of_qa_to_retrieve,
        query_filter=models.Filter(
            must=[
                models.FieldCondition(
                    key="is_impossible",
                    match=models.MatchValue(
                        value=False,
                    ),
                ),
            ],
        )
    )

    # Query Qdrant for similar questions that are IMPOSSIBLE to answer
    q2 = qdrant_client.search(
        collection_name=collection_name,
        query_vector=query_embedding,
        query_filter=models.Filter(
            must=[
                models.FieldCondition(
                    key="is_impossible",
                    match=models.MatchValue(
                        value=True,
                    ),
                ),
            ]
        ),
        with_payload=True,
        limit=num_of_qa_to_retrieve,
    )


    instruction = """Answer the following Question based on the Context only. Only answer from the Context. If you don't know the answer, say 'I don't know'.\n\n"""
    # If there is a next best question, add it to the prompt
    
    def q_to_prompt(q):
        question, context = q.payload["question"], q.payload["context"]
        answer = q.payload["answers"][0] if len(q.payload["answers"]) > 0 else "I don't know"
        return [
            {
                "role": "user", 
                "content": f"""Question: {question}\n\nContext: {context}\n\nAnswer:"""
            },
            {"role": "assistant", "content": answer},
        ]

    rag_prompt = []
    
    if len(q1) >= 1:
        rag_prompt += q_to_prompt(q1[1])
    if len(q2) >= 1:
        rag_prompt += q_to_prompt(q2[1])
    if len(q1) >= 1:
        rag_prompt += q_to_prompt(q1[2])
    
    

    rag_prompt += [
        {
            "role": "user",
            "content": f"""Question: {query}\n\nContext: {row_context}\n\nAnswer:"""
        },
    ]

    rag_prompt = [{"role": "system", "content": instruction}] + rag_prompt
    return rag_prompt
```


```python
# ⏰ Time: 2 min
train_sample["few_shot_prompt"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)
```

## 7. Fine-Tuning OpenAI Model with Qdrant

### 7.1 Upload the Fine-Tuning Data to OpenAI


```python
# Prepare the OpenAI File format i.e. JSONL from train_sample
def dataframe_to_jsonl(df):
    def create_jsonl_entry(row):
        messages = row["few_shot_prompt"]
        return json.dumps({"messages": messages})

    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)
    return "\n".join(jsonl_output)

with open("local_cache/100_train_few_shot.jsonl", "w") as f:
    f.write(dataframe_to_jsonl(train_sample))
```

### 7.2 Fine-Tune the Model

⏰ **Time to run: ~15-30 minutes**


key: _node_content
string_value:
{"id_": "beee9ec9-e146-4400-9276-d42a23938c9c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "cb51689e-c778-48a8-b989-4d0d64089391", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "9865e38d5456132fe96d9141110eb6304b7823ee436b4b3e2d7397a9eb0213c1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d0380afd-88a6-4557-8b41-0703f50fb295", "node_type": "1", "metadata": {}, "hash": "9d8addfcbc9314d55d4b1e2305bb2c3963226c07b97cea868b80904a2ca08f3d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 34894, "end_char_idx": 35850, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# ⏰ Time: 2 min
train_sample["few_shot_prompt"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)
```

## 7. Fine-Tuning OpenAI Model with Qdrant

### 7.1 Upload the Fine-Tuning Data to OpenAI


```python
# Prepare the OpenAI File format i.e. JSONL from train_sample
def dataframe_to_jsonl(df):
    def create_jsonl_entry(row):
        messages = row["few_shot_prompt"]
        return json.dumps({"messages": messages})

    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)
    return "\n".join(jsonl_output)

with open("local_cache/100_train_few_shot.jsonl", "w") as f:
    f.write(dataframe_to_jsonl(train_sample))
```

### 7.2 Fine-Tune the Model

⏰ **Time to run: ~15-30 minutes**


```python
fine_tuner = OpenAIFineTuner(
        training_file_path="local_cache/100_train_few_shot.jsonl",
        model_name="gpt-3.5-turbo",
        suffix="trnfewshot20230907"
    )

model_id = fine_tuner.fine_tune_model()
model_id
```


key: _node_content
string_value:
{"id_": "d0380afd-88a6-4557-8b41-0703f50fb295", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "beee9ec9-e146-4400-9276-d42a23938c9c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "570750d8ebd0a544ecea669c6750312b2f26db2f9a12b4d90e5f40e901334027", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b4dcdf5e-bc4b-48d3-b050-08163881a1c5", "node_type": "1", "metadata": {}, "hash": "9c3ba18dc6583179f20ffcaf5152e129438a1d3b4a7cb1d090f2024814c6cb41", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 34894, "end_char_idx": 38379, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# ⏰ Time: 2 min
train_sample["few_shot_prompt"] = train_sample.progress_apply(get_few_shot_prompt, axis=1)
```

## 7. Fine-Tuning OpenAI Model with Qdrant

### 7.1 Upload the Fine-Tuning Data to OpenAI


```python
# Prepare the OpenAI File format i.e. JSONL from train_sample
def dataframe_to_jsonl(df):
    def create_jsonl_entry(row):
        messages = row["few_shot_prompt"]
        return json.dumps({"messages": messages})

    jsonl_output = df.progress_apply(create_jsonl_entry, axis=1)
    return "\n".join(jsonl_output)

with open("local_cache/100_train_few_shot.jsonl", "w") as f:
    f.write(dataframe_to_jsonl(train_sample))
```

### 7.2 Fine-Tune the Model

⏰ **Time to run: ~15-30 minutes**


```python
fine_tuner = OpenAIFineTuner(
        training_file_path="local_cache/100_train_few_shot.jsonl",
        model_name="gpt-3.5-turbo",
        suffix="trnfewshot20230907"
    )

model_id = fine_tuner.fine_tune_model()
model_id
```


```python
# Let's try this out
completion = client.chat.completions.create(
    model=model_id,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {
            "role": "user",
            "content": "Can you answer the following question based on the given context? If not, say, I don't know:\n\nQuestion: What is the capital of France?\n\nContext: The capital of Mars is Gaia. Answer:",
        },
        {
            "role": "assistant",
            "content": "I don't know",
        },
        {
            "role": "user",
            "content": "Question: Where did Maharana Pratap die?\n\nContext: Rana Pratap's defiance of the mighty Mughal empire, almost alone and unaided by the other Rajput states, constitute a glorious saga of Rajput valour and the spirit of self sacrifice for cherished principles. Rana Pratap's methods of guerrilla warfare was later elaborated further by Malik Ambar, the Deccani general, and by Emperor Shivaji.\nAnswer:",
        },
        {
            "role": "assistant",
            "content": "I don't know",
        },
        {
            "role": "user",
            "content": "Question: Who did Rana Pratap fight against?\n\nContext: In stark contrast to other Rajput rulers who accommodated and formed alliances with the various Muslim dynasties in the subcontinent, by the time Pratap ascended to the throne, Mewar was going through a long standing conflict with the Mughals which started with the defeat of his grandfather Rana Sanga in the Battle of Khanwa in 1527 and continued with the defeat of his father Udai Singh II in Siege of Chittorgarh in 1568. Pratap Singh, gained distinction for his refusal to form any political alliance with the Mughal Empire and his resistance to Muslim domination. The conflicts between Pratap Singh and Akbar led to the Battle of Haldighati. Answer:",
        },
        {
            "role": "assistant",
            "content": "Akbar",
        },
        {
            "role": "user",
            "content": "Question: Which state is Chittorgarh in?\n\nContext: Chittorgarh, located in the southern part of the state of Rajasthan, 233 km (144.8 mi) from Ajmer, midway between Delhi and Mumbai on the National Highway 8 (India) in the road network of Golden Quadrilateral. Chittorgarh is situated where National Highways No. 76 & 79 intersect. Answer:",
        },
    ],
)
print("Correct Answer: Rajasthan\nModel Answer:")
print(completion.choices[0].message)
```

⏰ **Time to run: 5-15 min**


key: _node_content
string_value:
{"id_": "b4dcdf5e-bc4b-48d3-b050-08163881a1c5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d0380afd-88a6-4557-8b41-0703f50fb295", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "776de9eee57b20b92a1e30637c48c98a2a4e7be10645d8b07f1b9985ad30be14", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "15bd3fcf-0240-4e1f-91ec-677224acd633", "node_type": "1", "metadata": {}, "hash": "dd4d342b76d7449e06448f34fbf4c0759927b035843950ff2b0ed8fddc98b52b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 38382, "end_char_idx": 39938, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
df["ft_generated_answer_few_shot"] = df.progress_apply(answer_question, model=model_id, prompt_func=get_few_shot_prompt, axis=1)
df.to_json("local_cache/100_val_ft_few_shot.json", orient="records", lines=True)
```

## 8. Evaluation

But how well does the model perform? Let's compare the results from the 3 different models we've looked at so far:


```python
evaluator = Evaluator(df)
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer", "ft_generated_answer_few_shot"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned", "Fine-Tuned with Few-Shot"])
```


    
![png](output_56_0.png)
    


This is quite amazing -- we're able to get the best of both worlds! We're able to get the model to be both correct and conservative: 

1. The model is correct 83% of the time -- this is the same as the base model
2. The model gives the wrong answer only 8% of the time -- down from 17% with the base model

Next, let's look at the hallucinations. We want to reduce the hallucinations, but not at the cost of correctness. We want to strike a balance between the two. We've struck a good balance here:

1. The model hallucinates 53% of the time -- down from 100% with the base model
2. The model says "I don't know" 47% of the time -- up from NEVER with the base model


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer", "ft_generated_answer_few_shot"], scenario="idk_expected", nice_names=["Baseline", "Fine-Tuned", "Fine-Tuned with Few-Shot"])
```


    
![png](output_59_0.png)


key: _node_content
string_value:
{"id_": "15bd3fcf-0240-4e1f-91ec-677224acd633", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b4dcdf5e-bc4b-48d3-b050-08163881a1c5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "ff51c1f6e1e8431978f5784e801eae54546d282b95e263f1e3c1008de0c4c447", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "33e8977e-4dd2-489f-9df2-2878b6a06c43", "node_type": "1", "metadata": {}, "hash": "83efd75536847839db9dd72e908ed395701d3829bd31a091880bcfe2b59e776e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 38742, "end_char_idx": 41900, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
evaluator = Evaluator(df)
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer", "ft_generated_answer_few_shot"], scenario="answer_expected", nice_names=["Baseline", "Fine-Tuned", "Fine-Tuned with Few-Shot"])
```


    
![png](output_56_0.png)
    


This is quite amazing -- we're able to get the best of both worlds! We're able to get the model to be both correct and conservative: 

1. The model is correct 83% of the time -- this is the same as the base model
2. The model gives the wrong answer only 8% of the time -- down from 17% with the base model

Next, let's look at the hallucinations. We want to reduce the hallucinations, but not at the cost of correctness. We want to strike a balance between the two. We've struck a good balance here:

1. The model hallucinates 53% of the time -- down from 100% with the base model
2. The model says "I don't know" 47% of the time -- up from NEVER with the base model


```python
evaluator.plot_model_comparison(["generated_answer", "ft_generated_answer", "ft_generated_answer_few_shot"], scenario="idk_expected", nice_names=["Baseline", "Fine-Tuned", "Fine-Tuned with Few-Shot"])
```


    
![png](output_59_0.png)
    


Few Shot Fine-Tuning with Qdrant is a great way to control and steer the performance of your RAG system. Here, we made the model less conservative compared to zero shot and more confident by using Qdrant to find similar questions. 

You can also use Qdrant to make the model more conservative. We did this by giving examples of questions where the answer is not present in the context.  
This is biasing the model to say "I don't know" more often. 

Similarly, one can also use Qdrant to make the model more confident by giving examples of questions where the answer is present in the context. This biases the model to give an answer more often. The trade-off is that the model will also hallucinate more often.

You can make this trade off by adjusting the training data: distribution of questions and examples, as well as the kind and number of examples you retrieve from Qdrant.

## 9. Conclusion

In this notebook, we've demonstrated how to fine-tune OpenAI models for specific use-cases. We've also demonstrated how to use Qdrant and Few-Shot Learning to improve the performance of the model.

### Aggregate Results

So far, we've looked at the results for each scenario separately, i.e. each scenario summed to 100. Let's look at the results as an aggregate to get a broader sense of how the model is performing:

| Category | Base | Fine-Tuned | Fine-Tuned with Qdrant |
| --- | --- | --- | --- |
| Correct | 44% | 32% | 44% |
| Skipped | 0% | 18% | 5% |
| Wrong | 9% | 3% | 4% |
| Hallucination | 47% | 7% | 25% |
| I don't know | 0% | 40% | 22% |

### Observations

#### Compared to base model
1. The few shot fine-tuned with Qdrant model is as good as the base model at answering questions where the answer is present in the context. 
2. The few shot fine-tuned with Qdrant model is better at saying "I don't know" when the answer is not present in the context.
3. The few shot fine-tuned with Qdrant model is better at reducing hallucinations.


key: _node_content
string_value:
{"id_": "33e8977e-4dd2-489f-9df2-2878b6a06c43", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "725c64be-5bf2-4a54-8b2e-303f389e1a50", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c3d4e60a5301ff9c4065835cfa44463ecad1db6c2912cad00dd22578ffaf226d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "15bd3fcf-0240-4e1f-91ec-677224acd633", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/fine-tuned_qa/ft_retrieval_augmented_generation_qdrant.ipynb", "title": "Fine-Tuning OpenAI Models for Retrieval Augmented Generation (RAG) with Qdrant and Few-Shot Learning", "tokens": 10450, "retrieve_doc": false, "source": "openai_cookbooks"}, "hash": "c055d611d640f319436052228cccdc09f280b9f812ae107ebed5c9d1158545d1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 41903, "end_char_idx": 42696, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
#### Compared to fine-tuned model
1. The few shot fine-tuned with Qdrant model gets more correct answers than the fine-tuned model: **83% of the questions are answered correctly vs 60%** for the fine-tuned model
2. The few shot fine-tuned with Qdrant model is better at deciding when to say "I don't know" when the answer is not present in the context. **34% skip rate for the plain fine-tuning mode, vs 9% for the few shot fine-tuned with Qdrant model**


Now, you should be able to:

1. Notice the trade-offs between number of correct answers and hallucinations -- and how training dataset choice influences that!
2. Fine-tune OpenAI models for specific use-cases and use Qdrant to improve the performance of your RAG model
3. Get started on how to evaluate the performance of your RAG model


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb


key: _node_content
string_value:
{"id_": "8d269f20-ccf3-4750-a2b8-68e1f4686829", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1cb7fc44-2104-4083-ac9d-44bd7e033b85", "node_type": "1", "metadata": {}, "hash": "745a03d972c117d89502888561536d0cd90c1067ef62b129e3757c30418a3f24", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3332, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to use GPT-4 Vision with Function Calling 

The new GPT-4 Turbo model, available as gpt-4-turbo-2024-04-09 as of April 2024, now enables function calling with vision capabilities, better reasoning and a knowledge cutoff date of Dec 2023. Using images with function calling will unlock multimodal use cases and the ability to use reasoning, allowing you to go beyond OCR and image descriptions.

We will go through two examples to demonstrate the use of function calling with GPT-4 Turbo with Vision:

1. Simulating a customer service assistant for delivery exception support
2. Analyzing an organizational chart to extract employee information

### Installation and Setup


```python
!pip install pymupdf --quiet
!pip install openai --quiet
!pip install matplotlib --quiet
# instructor makes it easy to work with function calling
!pip install instructor --quiet
```


```python
import base64
import os
from enum import Enum
from io import BytesIO
from typing import Iterable
from typing import List
from typing import Literal, Optional

import fitz
# Instructor is powered by Pydantic, which is powered by type hints. Schema validation, prompting is controlled by type annotations
import instructor
import matplotlib.pyplot as plt
import pandas as pd
from IPython.display import display
from PIL import Image
from openai import OpenAI
from pydantic import BaseModel, Field
```

## 1. Simulating a customer service assistant for delivery exception support
We will simulate a customer service assistant for a delivery service that is equipped to analyze images of packages. The assistant will perform the following actions based on the image analysis:
- If a package appears damaged in the image, automatically process a refund according to policy.
- If the package looks wet, initiate a replacement.
- If the package appears normal and not damaged, escalate to an agent.

Let's look at the sample images of packages that the customer service assistant will analyze to determine the appropriate action. We will encode the images as base64 strings for processing by the model.


```python
# Function to encode the image as base64
def encode_image(image_path: str):
    # check if the image exists
    if not os.path.exists(image_path):
        raise FileNotFoundError(f"Image file not found: {image_path}")
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode('utf-8')


# Sample images for testing
image_dir = "images"

# encode all images within the directory
image_files = os.listdir(image_dir)
image_data = {}
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    # encode the image with key as the image file name
    image_data[image_file.split('.')[0]] = encode_image(image_path)
    print(f"Encoded image: {image_file}")


def display_images(image_data: dict):
    fig, axs = plt.subplots(1, 3, figsize=(18, 6))
    for i, (key, value) in enumerate(image_data.items()):
        img = Image.open(BytesIO(base64.b64decode(value)))
        ax = axs[i]
        ax.imshow(img)
        ax.axis("off")
        ax.set_title(key)
    plt.tight_layout()
    plt.show()


display_images(image_data)
```

    Encoded image: wet_package.jpg
    Encoded image: damaged_package.jpg
    Encoded image: normal_package.jpg



    
![png](output_6_1.png)


key: _node_content
string_value:
{"id_": "1cb7fc44-2104-4083-ac9d-44bd7e033b85", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8d269f20-ccf3-4750-a2b8-68e1f4686829", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "eb95dec29c0b405d5b4bdbb826ad97a7a7318a3aac39b30ef15d34a7e575a7af", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "19902666-7843-4d1c-a63d-7947560d36aa", "node_type": "1", "metadata": {}, "hash": "d3d45c8c1b85f18165f8e055d7c526a63c9ad5a0c26532dfbd2b013df157db59", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2082, "end_char_idx": 3894, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Function to encode the image as base64
def encode_image(image_path: str):
    # check if the image exists
    if not os.path.exists(image_path):
        raise FileNotFoundError(f"Image file not found: {image_path}")
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode('utf-8')


# Sample images for testing
image_dir = "images"

# encode all images within the directory
image_files = os.listdir(image_dir)
image_data = {}
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    # encode the image with key as the image file name
    image_data[image_file.split('.')[0]] = encode_image(image_path)
    print(f"Encoded image: {image_file}")


def display_images(image_data: dict):
    fig, axs = plt.subplots(1, 3, figsize=(18, 6))
    for i, (key, value) in enumerate(image_data.items()):
        img = Image.open(BytesIO(base64.b64decode(value)))
        ax = axs[i]
        ax.imshow(img)
        ax.axis("off")
        ax.set_title(key)
    plt.tight_layout()
    plt.show()


display_images(image_data)
```

    Encoded image: wet_package.jpg
    Encoded image: damaged_package.jpg
    Encoded image: normal_package.jpg



    
![png](output_6_1.png)
    


We have successfully encoded the sample images as base64 strings and displayed them. The customer service assistant will analyze these images to determine the appropriate action based on the package condition.

Let's now define the functions/tools for order processing, such as escalating an order to an agent, refunding an order, and replacing an order. We will create placeholder functions to simulate the processing of these actions based on the identified tools. We will be using Pydantic models to define the structure of the data for order actions.


key: _node_content
string_value:
{"id_": "19902666-7843-4d1c-a63d-7947560d36aa", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1cb7fc44-2104-4083-ac9d-44bd7e033b85", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d1c6f555131f1198b7d972d95fa39784bc9dd3ca5bcf54d69425c421acabae96", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "536ec321-e210-4bfc-8f8a-dac65bd7032a", "node_type": "1", "metadata": {}, "hash": "601b0f363b96725fb8e2aefc4305a30e273ca81d01b026fe6eca2561b435530a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2426, "end_char_idx": 6996, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Sample images for testing
image_dir = "images"

# encode all images within the directory
image_files = os.listdir(image_dir)
image_data = {}
for image_file in image_files:
    image_path = os.path.join(image_dir, image_file)
    # encode the image with key as the image file name
    image_data[image_file.split('.')[0]] = encode_image(image_path)
    print(f"Encoded image: {image_file}")


def display_images(image_data: dict):
    fig, axs = plt.subplots(1, 3, figsize=(18, 6))
    for i, (key, value) in enumerate(image_data.items()):
        img = Image.open(BytesIO(base64.b64decode(value)))
        ax = axs[i]
        ax.imshow(img)
        ax.axis("off")
        ax.set_title(key)
    plt.tight_layout()
    plt.show()


display_images(image_data)
```

    Encoded image: wet_package.jpg
    Encoded image: damaged_package.jpg
    Encoded image: normal_package.jpg



    
![png](output_6_1.png)
    


We have successfully encoded the sample images as base64 strings and displayed them. The customer service assistant will analyze these images to determine the appropriate action based on the package condition.

Let's now define the functions/tools for order processing, such as escalating an order to an agent, refunding an order, and replacing an order. We will create placeholder functions to simulate the processing of these actions based on the identified tools. We will be using Pydantic models to define the structure of the data for order actions.



```python
MODEL = "gpt-4-turbo-2024-04-09"

class Order(BaseModel):
    """Represents an order with details such as order ID, customer name, product name, price, status, and delivery date."""
    order_id: str = Field(..., description="The unique identifier of the order")
    product_name: str = Field(..., description="The name of the product")
    price: float = Field(..., description="The price of the product")
    status: str = Field(..., description="The status of the order")
    delivery_date: str = Field(..., description="The delivery date of the order")
# Placeholder functions for order processing

def get_order_details(order_id):
    # Placeholder function to retrieve order details based on the order ID
    return Order(
        order_id=order_id,
        product_name="Product X",
        price=100.0,
        status="Delivered",
        delivery_date="2024-04-10",
    )

def escalate_to_agent(order: Order, message: str):
    # Placeholder function to escalate the order to a human agent
    return f"Order {order.order_id} has been escalated to an agent with message: `{message}`"

def refund_order(order: Order):
    # Placeholder function to process a refund for the order
    return f"Order {order.order_id} has been refunded successfully."

def replace_order(order: Order):
    # Placeholder function to replace the order with a new one
    return f"Order {order.order_id} has been replaced with a new order."

class FunctionCallBase(BaseModel):
    rationale: Optional[str] = Field(..., description="The reason for the action.")
    image_description: Optional[str] = Field(
        ..., description="The detailed description of the package image."
    )
    action: Literal["escalate_to_agent", "replace_order", "refund_order"]
    message: Optional[str] = Field(
        ...,
        description="The message to be escalated to the agent if action is escalate_to_agent",
    )
    # Placeholder functions to process the action based on the order ID
    def __call__(self, order_id):
        order: Order = get_order_details(order_id=order_id)
        if self.action == "escalate_to_agent":
            return escalate_to_agent(order, self.message)
        if self.action == "replace_order":
            return replace_order(order)
        if self.action == "refund_order":
            return refund_order(order)

class EscalateToAgent(FunctionCallBase):
    """Escalate to an agent for further assistance."""
    pass

class OrderActionBase(FunctionCallBase):
    pass

class ReplaceOrder(OrderActionBase):
    """Tool call to replace an order."""
    pass

class RefundOrder(OrderActionBase):
    """Tool call to refund an order."""
    pass
```

### Simulating user messages and processing the package images

We will simulate user messages containing the package images and process the images using the GPT-4 Turbo with Vision model. The model will identify the appropriate tool call based on the image analysis and the predefined actions for damaged, wet, or normal packages. We will then process the identified action based on the order ID and display the results.


key: _node_content
string_value:
{"id_": "536ec321-e210-4bfc-8f8a-dac65bd7032a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "19902666-7843-4d1c-a63d-7947560d36aa", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "1f1f0aaa93b8275f4c4d32ed3d7246ca75a7f7e5335920b15e9926effdc5dddf", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "32530480-d97f-45d0-b531-4bf710b1d0aa", "node_type": "1", "metadata": {}, "hash": "1d31a0d166c9e36b48e6c7942431e27b88965cc6df32228d4bcf9656f4bdeca2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6999, "end_char_idx": 8706, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# extract the tool call from the response
ORDER_ID = "12345"  # Placeholder order ID for testing
INSTRUCTION_PROMPT = "You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools!"

def delivery_exception_support_handler(test_image: str):
    payload = {
        "model": MODEL,
        "response_model": Iterable[RefundOrder | ReplaceOrder | EscalateToAgent],
        "tool_choice": "auto",  # automatically select the tool based on the context
        "temperature": 0.0,  # for less diversity in responses
        "seed": 123,  # Set a seed for reproducibility
    }
    payload["messages"] = [
        {
            "role": "user",
            "content": INSTRUCTION_PROMPT,
        },
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{image_data[test_image]}"
                    }
                },
            ],
        }
    ]
    function_calls = instructor.from_openai(
        OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS
    ).chat.completions.create(**payload)
    for tool in function_calls:
        print(f"- Tool call: {tool.action} for provided img: {test_image}")
        print(f"- Parameters: {tool}")
        print(f">> Action result: {tool(ORDER_ID)}")
        return tool


key: _node_content
string_value:
{"id_": "32530480-d97f-45d0-b531-4bf710b1d0aa", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "536ec321-e210-4bfc-8f8a-dac65bd7032a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dc526499f2c39f74dc723a92494bf45ffffed42f225fce30fbb98e15a179df8b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "29546062-d691-4574-a0bc-12d7bf1bd0c0", "node_type": "1", "metadata": {}, "hash": "0161fce6c3cc5d65361fe0b946bc1e4742fcece4127c254e69f4ff93f4535a45", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6999, "end_char_idx": 11389, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# extract the tool call from the response
ORDER_ID = "12345"  # Placeholder order ID for testing
INSTRUCTION_PROMPT = "You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools!"

def delivery_exception_support_handler(test_image: str):
    payload = {
        "model": MODEL,
        "response_model": Iterable[RefundOrder | ReplaceOrder | EscalateToAgent],
        "tool_choice": "auto",  # automatically select the tool based on the context
        "temperature": 0.0,  # for less diversity in responses
        "seed": 123,  # Set a seed for reproducibility
    }
    payload["messages"] = [
        {
            "role": "user",
            "content": INSTRUCTION_PROMPT,
        },
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{image_data[test_image]}"
                    }
                },
            ],
        }
    ]
    function_calls = instructor.from_openai(
        OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS
    ).chat.completions.create(**payload)
    for tool in function_calls:
        print(f"- Tool call: {tool.action} for provided img: {test_image}")
        print(f"- Parameters: {tool}")
        print(f">> Action result: {tool(ORDER_ID)}")
        return tool


print("Processing delivery exception support for different package images...")

print("\n===================== Simulating user message 1 =====================")
assert delivery_exception_support_handler("damaged_package").action == "refund_order"

print("\n===================== Simulating user message 2 =====================")
assert delivery_exception_support_handler("normal_package").action == "escalate_to_agent"

print("\n===================== Simulating user message 3 =====================")
assert delivery_exception_support_handler("wet_package").action == "replace_order"
```

    Processing delivery exception support for different package images...
    
    ===================== Simulating user message 1 =====================
    - Tool call: refund_order for provided img: damaged_package
    - Parameters: rationale='The package is visibly damaged with significant tears and crushing, indicating potential harm to the contents.' image_description='The package in the image shows extensive damage, including deep creases and tears in the cardboard. The package is also wrapped with extra tape, suggesting prior attempts to secure it after damage.' action='refund_order' message=None
    >> Action result: Order 12345 has been refunded successfully.
    
    ===================== Simulating user message 2 =====================
    - Tool call: escalate_to_agent for provided img: normal_package
    - Parameters: rationale='The package appears normal and not damaged, requiring further assistance for any potential issues not visible in the image.' image_description='A cardboard box on a wooden floor, appearing intact and undamaged, with no visible signs of wear, tear, or wetness.' action='escalate_to_agent' message='Please review this package for any issues not visible in the image. The package appears normal and undamaged.'
    >> Action result: Order 12345 has been escalated to an agent with message: `Please review this package for any issues not visible in the image. The package appears normal and undamaged.`
    
    ===================== Simulating user message 3 =====================
    - Tool call: replace_order for provided img: wet_package
    - Parameters: rationale='The package appears wet, which may have compromised the contents, especially since it is labeled as fragile.' image_description="The package in the image shows significant wetness on the top surface, indicating potential water damage. The box is labeled 'FRAGILE', which suggests that the contents are delicate and may be more susceptible to damage from moisture." action='replace_order' message=None
    >> Action result: Order 12345 has been replaced with a new order.


key: _node_content
string_value:
{"id_": "29546062-d691-4574-a0bc-12d7bf1bd0c0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "32530480-d97f-45d0-b531-4bf710b1d0aa", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "aa1077ed3a231e949968872ba72a9e6105ca7767e76fb2e4ef8f9383eff34089", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6a2061b6-cbc2-4d23-9763-f8429403a1f7", "node_type": "1", "metadata": {}, "hash": "a74d0c370790e2b8f38e6374975e7130d77095e5d57d70256b63e9fd766b3d06", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11392, "end_char_idx": 14076, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 2.  Analyzing an organizational chart to extract employee information

For the second example, we will analyze an organizational chart image to extract employee information, such as employee names, roles, managers, and manager roles. We will use GPT-4 Turbo with Vision to process the organizational chart image and extract structured data about the employees in the organization. Indeed, function calling lets us go beyond OCR to actually deduce and translate hierarchical relationships within the chart.

We will start with a sample organizational chart in PDF format that we want to analyze and convert the first page of the PDF to a JPEG image for analysis.


```python
# Function to convert a single page PDF page to a JPEG image
def convert_pdf_page_to_jpg(pdf_path: str, output_path: str, page_number=0):
    if not os.path.exists(pdf_path):
        raise FileNotFoundError(f"PDF file not found: {pdf_path}")
    doc = fitz.open(pdf_path)
    page = doc.load_page(page_number)  # 0 is the first page
    pix = page.get_pixmap()
    # Save the pixmap as a JPEG
    pix.save(output_path)


def display_img_local(image_path: str):
    img = Image.open(image_path)
    display(img)


pdf_path = 'data/org-chart-sample.pdf'
output_path = 'org-chart-sample.jpg'

convert_pdf_page_to_jpg(pdf_path, output_path)
display_img_local(output_path)
```


    
![png](output_12_0.png)
    


The organizational chart image has been successfully extracted from the PDF file and displayed. Let's now define a function to analyze the organizational chart image using the new GPT4 Turbo with Vision. The function will extract information about the employees, their roles, and their managers from the image. We will use function/tool calling to specify the input parameters for the organizational structure, such as the employee name, role, and manager's name and role. We will use Pydantic models to define the structure of the data.



```python
base64_img = encode_image(output_path)

class RoleEnum(str, Enum):
    """Defines possible roles within an organization."""
    CEO = "CEO"
    CTO = "CTO"
    CFO = "CFO"
    COO = "COO"
    EMPLOYEE = "Employee"
    MANAGER = "Manager"
    INTERN = "Intern"
    OTHER = "Other"

class Employee(BaseModel):
    """Represents an employee, including their name, role, and optional manager information."""
    employee_name: str = Field(..., description="The name of the employee")
    role: RoleEnum = Field(..., description="The role of the employee")
    manager_name: Optional[str] = Field(None, description="The manager's name, if applicable")
    manager_role: Optional[RoleEnum] = Field(None, description="The manager's role, if applicable")


key: _node_content
string_value:
{"id_": "6a2061b6-cbc2-4d23-9763-f8429403a1f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "29546062-d691-4574-a0bc-12d7bf1bd0c0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ce6c5165ed04b2fc741494e193020bde6222e6e5de6ab9752c59f95d3bbe7ecd", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5031f986-d901-4ea7-98e8-2701c6c7d4a3", "node_type": "1", "metadata": {}, "hash": "8afa97cc5acd2cc2c8a615f65a45c5d50bdf2d608544d1fb1f7bbef4dd07e52b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12490, "end_char_idx": 15969, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
def display_img_local(image_path: str):
    img = Image.open(image_path)
    display(img)


pdf_path = 'data/org-chart-sample.pdf'
output_path = 'org-chart-sample.jpg'

convert_pdf_page_to_jpg(pdf_path, output_path)
display_img_local(output_path)
```


    
![png](output_12_0.png)
    


The organizational chart image has been successfully extracted from the PDF file and displayed. Let's now define a function to analyze the organizational chart image using the new GPT4 Turbo with Vision. The function will extract information about the employees, their roles, and their managers from the image. We will use function/tool calling to specify the input parameters for the organizational structure, such as the employee name, role, and manager's name and role. We will use Pydantic models to define the structure of the data.



```python
base64_img = encode_image(output_path)

class RoleEnum(str, Enum):
    """Defines possible roles within an organization."""
    CEO = "CEO"
    CTO = "CTO"
    CFO = "CFO"
    COO = "COO"
    EMPLOYEE = "Employee"
    MANAGER = "Manager"
    INTERN = "Intern"
    OTHER = "Other"

class Employee(BaseModel):
    """Represents an employee, including their name, role, and optional manager information."""
    employee_name: str = Field(..., description="The name of the employee")
    role: RoleEnum = Field(..., description="The role of the employee")
    manager_name: Optional[str] = Field(None, description="The manager's name, if applicable")
    manager_role: Optional[RoleEnum] = Field(None, description="The manager's role, if applicable")


class EmployeeList(BaseModel):
    """A list of employees within the organizational structure."""
    employees: List[Employee] = Field(..., description="A list of employees")

def parse_orgchart(base64_img: str) -> EmployeeList:
    response = instructor.from_openai(OpenAI()).chat.completions.create(
        model='gpt-4-turbo',
        response_model=EmployeeList,
        messages=[
            {
                "role": "user",
                "content": 'Analyze the given organizational chart and very carefully extract the information.',
            },
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{base64_img}"
                        }
                    },
                ],
            }
        ],
    )
    return response
```

Now, we will define a function to parse the response from GPT-4 Turbo with Vision and extract the employee data. We will tabulate the extracted data for easy visualization. Please note that the accuracy of the extracted data may vary based on the complexity and clarity of the input image.


```python
# call the functions to analyze the organizational chart and parse the response
result = parse_orgchart(base64_img)

# tabulate the extracted data
df = pd.DataFrame([{
    'employee_name': employee.employee_name,
    'role': employee.role.value,
    'manager_name': employee.manager_name,
    'manager_role': employee.manager_role.value if employee.manager_role else None
} for employee in result.employees])

display(df)
```


<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;


key: _node_content
string_value:
{"id_": "5031f986-d901-4ea7-98e8-2701c6c7d4a3", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6a2061b6-cbc2-4d23-9763-f8429403a1f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "6de0ce211bff2774ec83e80ae37710cae4eb9b30088172611d7ec25bcd604c7a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2419ae36-839e-4a18-9b7c-aec12b1e655a", "node_type": "1", "metadata": {}, "hash": "ece6b649218241031b7087b48f3aca6603da0a823f8531daa215ea826646111d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14079, "end_char_idx": 16658, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
class EmployeeList(BaseModel):
    """A list of employees within the organizational structure."""
    employees: List[Employee] = Field(..., description="A list of employees")

def parse_orgchart(base64_img: str) -> EmployeeList:
    response = instructor.from_openai(OpenAI()).chat.completions.create(
        model='gpt-4-turbo',
        response_model=EmployeeList,
        messages=[
            {
                "role": "user",
                "content": 'Analyze the given organizational chart and very carefully extract the information.',
            },
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{base64_img}"
                        }
                    },
                ],
            }
        ],
    )
    return response
```

Now, we will define a function to parse the response from GPT-4 Turbo with Vision and extract the employee data. We will tabulate the extracted data for easy visualization. Please note that the accuracy of the extracted data may vary based on the complexity and clarity of the input image.


```python
# call the functions to analyze the organizational chart and parse the response
result = parse_orgchart(base64_img)

# tabulate the extracted data
df = pd.DataFrame([{
    'employee_name': employee.employee_name,
    'role': employee.role.value,
    'manager_name': employee.manager_name,
    'manager_role': employee.manager_role.value if employee.manager_role else None
} for employee in result.employees])

display(df)
```


<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>employee_name</th>
      <th>role</th>
      <th>manager_name</th>
      <th>manager_role</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Juliana Silva</td>
      <td>CEO</td>
      <td>None</td>
      <td>None</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Kim Chun Hei</td>
      <td>CFO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Chad Gibbons</td>
      <td>CTO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Chiaki Sato</td>


key: _node_content
string_value:
{"id_": "2419ae36-839e-4a18-9b7c-aec12b1e655a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5031f986-d901-4ea7-98e8-2701c6c7d4a3", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "f51b16322cc6018674fe38a22b7f4c1f76d78510b893d952af302601f57a9aa6", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7127447e-bc65-4ae7-8634-3bf22c8a3a25", "node_type": "1", "metadata": {}, "hash": "1a83601e0883f8d7bd158bd92af26ca072184d6a5c88e490c4591e25bed6e27f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15914, "end_char_idx": 17294, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
}

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>employee_name</th>
      <th>role</th>
      <th>manager_name</th>
      <th>manager_role</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Juliana Silva</td>
      <td>CEO</td>
      <td>None</td>
      <td>None</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Kim Chun Hei</td>
      <td>CFO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Chad Gibbons</td>
      <td>CTO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Chiaki Sato</td>
      <td>COO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
      <td>Kim Chun Hei</td>
      <td>CFO</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Shawn Garcia</td>
      <td>Manager</td>
      <td>Chad Gibbons</td>
      <td>CTO</td>
    </tr>
    <tr>
      <th>6</th>
      <td>Aaron Loeb</td>
      <td>Manager</td>
      <td>Chiaki Sato</td>
      <td>COO</td>
    </tr>
    <tr>
      <th>7</th>
      <td>Drew Feig</td>
      <td>Employee</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>8</th>


key: _node_content
string_value:
{"id_": "7127447e-bc65-4ae7-8634-3bf22c8a3a25", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2419ae36-839e-4a18-9b7c-aec12b1e655a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "65e2eca3add52e7be4d52ace1d434031dead81c0df7e4f9ce99de5b32cbfa736", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8f3ae007-c037-4091-9dcb-7801564e102e", "node_type": "1", "metadata": {}, "hash": "969460755f127eb82048844676c1719f5c0e279955c7e0963406270c7455c8d1", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16638, "end_char_idx": 17968, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<td>Chiaki Sato</td>
      <td>COO</td>
      <td>Juliana Silva</td>
      <td>CEO</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
      <td>Kim Chun Hei</td>
      <td>CFO</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Shawn Garcia</td>
      <td>Manager</td>
      <td>Chad Gibbons</td>
      <td>CTO</td>
    </tr>
    <tr>
      <th>6</th>
      <td>Aaron Loeb</td>
      <td>Manager</td>
      <td>Chiaki Sato</td>
      <td>COO</td>
    </tr>
    <tr>
      <th>7</th>
      <td>Drew Feig</td>
      <td>Employee</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>8</th>
      <td>Richard Sanchez</td>
      <td>Employee</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>9</th>
      <td>Sacha Dubois</td>
      <td>Intern</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>10</th>
      <td>Olivia Wilson</td>
      <td>Employee</td>
      <td>Shawn Garcia</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>11</th>
      <td>Matt Zhang</td>
      <td>Intern</td>
      <td>Shawn Garcia</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>12</th>
      <td>Avery Davis</td>
      <td>Employee</td>
      <td>Aaron Loeb</td>
      <td>Manager</td>
    </tr>
    <tr>


key: _node_content
string_value:
{"id_": "8f3ae007-c037-4091-9dcb-7801564e102e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "c3e86346-8042-43f4-836c-7584c6f49c51", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "dd12f61a1673bab75775a08698bc39b904361ac81726c523ce42781ad5048b1c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7127447e-bc65-4ae7-8634-3bf22c8a3a25", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb", "title": "How to use GPT-4 Vision with Function Calling", "tokens": 4334, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "324b5399da40f34ea71171c394ebde9102acca5cdd7d35786a53385a288b8310", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 17263, "end_char_idx": 18712, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
</tr>
    <tr>
      <th>8</th>
      <td>Richard Sanchez</td>
      <td>Employee</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>9</th>
      <td>Sacha Dubois</td>
      <td>Intern</td>
      <td>Cahaya Dewi</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>10</th>
      <td>Olivia Wilson</td>
      <td>Employee</td>
      <td>Shawn Garcia</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>11</th>
      <td>Matt Zhang</td>
      <td>Intern</td>
      <td>Shawn Garcia</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>12</th>
      <td>Avery Davis</td>
      <td>Employee</td>
      <td>Aaron Loeb</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>13</th>
      <td>Harper Russo</td>
      <td>Employee</td>
      <td>Aaron Loeb</td>
      <td>Manager</td>
    </tr>
    <tr>
      <th>14</th>
      <td>Taylor Alonso</td>
      <td>Intern</td>
      <td>Aaron Loeb</td>
      <td>Manager</td>
    </tr>
  </tbody>
</table>
</div>


The extracted data from the organizational chart has been successfully parsed and displayed in a DataFrame. This approach allows us to leverage GPT-4 Turbo with Vision capabilities to extract structured information from images, such as organizational charts and diagrams, and process the data for further analysis. By using function calling, we can extend the functionality of multimodal models to perform specific tasks or call external functions.


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb


key: _node_content
string_value:
{"id_": "a91ef393-4096-4f7b-bd2c-66c96e1cc92e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb", "title": "Load the embeddings", "tokens": 506, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "5ff02a77-1c9f-4eed-a391-ed7d03e2b251", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_wandb.ipynb", "title": "Load the embeddings", "tokens": 506, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "9755f69133468856653acb6541fd6cd6611153750fa57cfc2357518380cf3dea", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2071, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Visualizing embeddings in W&B

We will upload the data to [Weights & Biases](http://wandb.ai) and use an [Embedding Projector](https://docs.wandb.ai/ref/app/features/panels/weave/embedding-projector) to visualize the embeddings using common dimension reduction algorithms like PCA, UMAP, and t-SNE. The dataset is created in the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb).

## What is Weights & Biases?

[Weights & Biases](http://wandb.ai) is a machine learning platform used by OpenAI and other ML teams to build better models faster. They use it to quickly track experiments, evaluate model performance, reproduce models, visualize results, and share findings with colleagues.

### 1. Log the data to W&B

We create a [W&B Table](https://docs.wandb.ai/guides/data-vis/log-tables) with the original data and the embeddings. Each review is a new row and the 1536 embedding floats are given their own column named `emb_{i}`.


```python
import pandas as pd
from sklearn.manifold import TSNE
import numpy as np
from ast import literal_eval

# Load the embeddings
datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"
df = pd.read_csv(datafile_path)

# Convert to a list of lists of floats
matrix = np.array(df.embedding.apply(literal_eval).to_list())
```


```python
import wandb

original_cols = df.columns[1:-1].tolist()
embedding_cols = ['emb_'+str(idx) for idx in range(len(matrix[0]))]
table_cols = original_cols + embedding_cols

with wandb.init(project='openai_embeddings'):
    table = wandb.Table(columns=table_cols)
    for i, row in enumerate(df.to_dict(orient="records")):
        original_data = [row[col_name] for col_name in original_cols]
        embedding_data = matrix[i].tolist()
        table.add_data(*(original_data + embedding_data))
    wandb.log({'openai_embedding_table': table})
```

### 2. Render as 2D Projection

After navigating to the W&B run link, we click the ⚙️ icon in the top right of the Table and change "Render As:" to "Combined 2D Projection". 

Example: http://wandb.me/openai_embeddings


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb


key: _node_content
string_value:
{"id_": "cb456b1a-0b5d-4e11-8bdf-56ff5bca2c44", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a119c892-c6f5-4231-ac5e-767882e38ceb", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4f84c2d9-9050-4c0a-9e13-717648b30f74", "node_type": "1", "metadata": {}, "hash": "5c238dd0605c3618695e3c739f33801a19df5a659132e2b7841934c4807a5fdf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 1733, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to automate tasks with functions (S3 bucket example)

This code demonstrates how to interact with ChatGPT functions to perform tasks related to Amazon S3 buckets. The notebook covers S3 bucket key functionalities such as running simple listing commands, searching for a specific file in all buckets, uploading a file to a bucket, and downloading a file from a bucket. The OpenAI Chat API understands the user instructions, generates the natural language responses, and extracts appropriate function calls based on the user's input.

**Requirements**:
To run the notebook generate AWS access key with S3 bucket writing permission and store them in a local environment file alongside the Openai key. The "`.env`" file format:
```
AWS_ACCESS_KEY_ID=<your-key>
AWS_SECRET_ACCESS_KEY=<your-key>
OPENAI_API_KEY=<your-key>
```


```python
! pip install openai
! pip install boto3
! pip install tenacity
! pip install python-dotenv
```


```python
from openai import OpenAI
import json
import boto3
import os
import datetime
from urllib.request import urlretrieve

# load environment variables
from dotenv import load_dotenv
load_dotenv() 
```




    True



## Initials


```python
OpenAI.api_key = os.environ.get("OPENAI_API_KEY")
GPT_MODEL = "gpt-3.5-turbo"
```


```python
# Optional - if you had issues loading the environment file, you can set the AWS values using the below code
# os.environ['AWS_ACCESS_KEY_ID'] = ''
# os.environ['AWS_SECRET_ACCESS_KEY'] = ''

# Create S3 client
s3_client = boto3.client('s3')

# Create openai client
client = OpenAI()
```

## Utilities

To connect user questions or commands to the appropriate function, we need to provide ChatGPT with the necessary function details and expected parameters.


key: _node_content
string_value:
{"id_": "4f84c2d9-9050-4c0a-9e13-717648b30f74", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a119c892-c6f5-4231-ac5e-767882e38ceb", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "cb456b1a-0b5d-4e11-8bdf-56ff5bca2c44", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "93ff23939b8b3257c38a002c1c93b5aac64304109c5698dcdbe90890fc0eaa84", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b619b9bc-5b03-4495-94fb-b634f085e7b6", "node_type": "1", "metadata": {}, "hash": "601087bcb95d65bb471ab51e817d00b8f2b784a8275e195e9023b61bb302d294", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 5303, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to automate tasks with functions (S3 bucket example)

This code demonstrates how to interact with ChatGPT functions to perform tasks related to Amazon S3 buckets. The notebook covers S3 bucket key functionalities such as running simple listing commands, searching for a specific file in all buckets, uploading a file to a bucket, and downloading a file from a bucket. The OpenAI Chat API understands the user instructions, generates the natural language responses, and extracts appropriate function calls based on the user's input.

**Requirements**:
To run the notebook generate AWS access key with S3 bucket writing permission and store them in a local environment file alongside the Openai key. The "`.env`" file format:
```
AWS_ACCESS_KEY_ID=<your-key>
AWS_SECRET_ACCESS_KEY=<your-key>
OPENAI_API_KEY=<your-key>
```


```python
! pip install openai
! pip install boto3
! pip install tenacity
! pip install python-dotenv
```


```python
from openai import OpenAI
import json
import boto3
import os
import datetime
from urllib.request import urlretrieve

# load environment variables
from dotenv import load_dotenv
load_dotenv() 
```




    True



## Initials


```python
OpenAI.api_key = os.environ.get("OPENAI_API_KEY")
GPT_MODEL = "gpt-3.5-turbo"
```


```python
# Optional - if you had issues loading the environment file, you can set the AWS values using the below code
# os.environ['AWS_ACCESS_KEY_ID'] = ''
# os.environ['AWS_SECRET_ACCESS_KEY'] = ''

# Create S3 client
s3_client = boto3.client('s3')

# Create openai client
client = OpenAI()
```

## Utilities

To connect user questions or commands to the appropriate function, we need to provide ChatGPT with the necessary function details and expected parameters.


```python
# Functions dict to pass S3 operations details for the GPT model
functions = [
    {   
        "type": "function",
        "function":{
            "name": "list_buckets",
            "description": "List all available S3 buckets",
            "parameters": {
                "type": "object",
                "properties": {}
            }
        }
    },
    {
        "type": "function",
        "function":{
            "name": "list_objects",
            "description": "List the objects or files inside a given S3 bucket",
            "parameters": {
                "type": "object",
                "properties": {
                    "bucket": {"type": "string", "description": "The name of the S3 bucket"},
                    "prefix": {"type": "string", "description": "The folder path in the S3 bucket"},
                },
                "required": ["bucket"],
            },
        }
    },
    {   
        "type": "function",
        "function":{
            "name": "download_file",
            "description": "Download a specific file from an S3 bucket to a local distribution folder.",
            "parameters": {
                "type": "object",
                "properties": {
                    "bucket": {"type": "string", "description": "The name of the S3 bucket"},
                    "key": {"type": "string", "description": "The path to the file inside the bucket"},
                    "directory": {"type": "string", "description": "The local destination directory to download the file, should be specificed by the user."},
                },
                "required": ["bucket", "key", "directory"],
            }
        }
    },
    {
        "type": "function",
        "function":{
            "name": "upload_file",
            "description": "Upload a file to an S3 bucket",
            "parameters": {
                "type": "object",
                "properties": {
                    "source": {"type": "string", "description": "The local source path or remote URL"},
                    "bucket": {"type": "string", "description": "The name of the S3 bucket"},
                    "key": {"type": "string", "description": "The path to the file inside the bucket"},
                    "is_remote_url": {"type": "boolean", "description": "Is the provided source a URL (True) or local path (False)"},
                },
                "required": ["source", "bucket", "key", "is_remote_url"],
            }
        }
    },
    {
        "type": "function",
        "function":{
            "name": "search_s3_objects",
            "description": "Search for a specific file name inside an S3 bucket",
            "parameters": {
                "type": "object",
                "properties": {
                    "search_name": {"type": "string", "description": "The name of the file you want to search for"},
                    "bucket": {"type": "string", "description": "The name of the S3 bucket"},
                    "prefix": {"type": "string", "description": "The folder path in the S3 bucket"},
                    "exact_match": {"type": "boolean", "description": "Set exact_match to True if the search should match the exact file name. Set exact_match to False to compare part of the file name string (the file contains)"}
                },
                "required": ["search_name"],
            },
        }
    }
]
```

Create helper functions to interact with the S3 service, such as listing buckets, listing objects, downloading and uploading files, and searching for specific files.


key: _node_content
string_value:
{"id_": "b619b9bc-5b03-4495-94fb-b634f085e7b6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a119c892-c6f5-4231-ac5e-767882e38ceb", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4f84c2d9-9050-4c0a-9e13-717648b30f74", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ae09f84e5de46a76abc534f5b1485f579006d62c1067bd884cff374dd7e64986", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "45852883-cdad-4b00-a25c-5444348663bd", "node_type": "1", "metadata": {}, "hash": "dc11f89f7730b179f1533250e1d70834a4a7ce12f54f9762df55c9ff31349b0d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5306, "end_char_idx": 8423, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def datetime_converter(obj):
    if isinstance(obj, datetime.datetime):
        return obj.isoformat()
    raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable")
```


```python
def list_buckets():
    response = s3_client.list_buckets()
    return json.dumps(response['Buckets'], default=datetime_converter)

def list_objects(bucket, prefix=''):
    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix)
    return json.dumps(response.get('Contents', []), default=datetime_converter)

def download_file(bucket, key, directory):
    
    filename = os.path.basename(key)
    
    # Resolve destination to the correct file path
    destination = os.path.join(directory, filename)
    
    s3_client.download_file(bucket, key, destination)
    return json.dumps({"status": "success", "bucket": bucket, "key": key, "destination": destination})

def upload_file(source, bucket, key, is_remote_url=False):
    if is_remote_url:
        file_name = os.path.basename(source)
        urlretrieve(source, file_name)
        source = file_name
       
    s3_client.upload_file(source, bucket, key)
    return json.dumps({"status": "success", "source": source, "bucket": bucket, "key": key})

def search_s3_objects(search_name, bucket=None, prefix='', exact_match=True):
    search_name = search_name.lower()
    
    if bucket is None:
        buckets_response = json.loads(list_buckets())
        buckets = [bucket_info["Name"] for bucket_info in buckets_response]
    else:
        buckets = [bucket]

    results = []

    for bucket_name in buckets:
        objects_response = json.loads(list_objects(bucket_name, prefix))
        if exact_match:
            bucket_results = [obj for obj in objects_response if search_name == obj['Key'].lower()]
        else:
            bucket_results = [obj for obj in objects_response if search_name in obj['Key'].lower()]

        if bucket_results:
            results.extend([{"Bucket": bucket_name, "Object": obj} for obj in bucket_results])

    return json.dumps(results)
```

The below dictionary connects the name with the function to use it for execution based on ChatGPT responses.


```python
available_functions = {
    "list_buckets": list_buckets,
    "list_objects": list_objects,
    "download_file": download_file,
    "upload_file": upload_file,
    "search_s3_objects": search_s3_objects
}
```

## ChatGPT


```python
def chat_completion_request(messages, functions=None, function_call='auto', 
                            model_name=GPT_MODEL):
    
    if functions is not None:
        return client.chat.completions.create(
            model=model_name,
            messages=messages,
            tools=functions,
            tool_choice=function_call)
    else:
        return client.chat.completions.create(
            model=model_name,
            messages=messages)
```

### Conversation flow

Create a main function for the chatbot, which takes user input, sends it to the OpenAI Chat API, receives a response, executes any function calls generated by the API, and returns a final response to the user.


key: _node_content
string_value:
{"id_": "45852883-cdad-4b00-a25c-5444348663bd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a119c892-c6f5-4231-ac5e-767882e38ceb", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b619b9bc-5b03-4495-94fb-b634f085e7b6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "10d0d36e63e0a1bab8138097d8ef723cc7cb0bd1f4eac2d66a7d61a385b1d910", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4b62200b-82cd-4fc4-a9b0-18104d931e56", "node_type": "1", "metadata": {}, "hash": "a0e08ba5183d28fe846d2380010a8c01ccd9e5d2fa2e3c7295f3f7ff485a7090", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7486, "end_char_idx": 10861, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
available_functions = {
    "list_buckets": list_buckets,
    "list_objects": list_objects,
    "download_file": download_file,
    "upload_file": upload_file,
    "search_s3_objects": search_s3_objects
}
```

## ChatGPT


```python
def chat_completion_request(messages, functions=None, function_call='auto', 
                            model_name=GPT_MODEL):
    
    if functions is not None:
        return client.chat.completions.create(
            model=model_name,
            messages=messages,
            tools=functions,
            tool_choice=function_call)
    else:
        return client.chat.completions.create(
            model=model_name,
            messages=messages)
```

### Conversation flow

Create a main function for the chatbot, which takes user input, sends it to the OpenAI Chat API, receives a response, executes any function calls generated by the API, and returns a final response to the user.


```python
def run_conversation(user_input, topic="S3 bucket functions.", is_log=False):

    system_message=f"Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous. If the user ask question not related to {topic} response your scope is {topic} only."
    
    messages = [{"role": "system", "content": system_message},
                {"role": "user", "content": user_input}]
    
    # Call the model to get a response
    response = chat_completion_request(messages, functions=functions)
    response_message = response.choices[0].message
    
    if is_log:
        print(response.choices)
    
    # check if GPT wanted to call a function
    if response_message.tool_calls:
        function_name = response_message.tool_calls[0].function.name
        function_args = json.loads(response_message.tool_calls[0].function.arguments)
        
        # Call the function
        function_response = available_functions[function_name](**function_args)
        
        # Add the response to the conversation
        messages.append(response_message)
        messages.append({
            "role": "tool",
            "content": function_response,
            "tool_call_id": response_message.tool_calls[0].id,
        })
        
        # Call the model again to summarize the results
        second_response = chat_completion_request(messages)
        final_message = second_response.choices[0].message.content
    else:
        final_message = response_message.content

    return final_message
```

### S3 bucket bot testing
In the following examples, make sure to replace the placeholders such as `<file_name>`, `<bucket_name>`, and `<directory_path>` with your specific values before execution.

#### Listing and searching

Let's start by listing all the available buckets.


```python
print(run_conversation('list my S3 buckets'))
```

You can ask the assistant to search for a specific file name either in all the buckets or in a specific one.


```python
search_file = '<file_name>'
print(run_conversation(f'search for a file {search_file} in all buckets'))
```


```python
search_word = '<file_name_part>'
bucket_name = '<bucket_name>'
print(run_conversation(f'search for a file contains {search_word} in {bucket_name}'))
```

The model is expected to clarify the ask from the user in case of ambiguity in the parameters values as described in the system message.


key: _node_content
string_value:
{"id_": "4b62200b-82cd-4fc4-a9b0-18104d931e56", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "a119c892-c6f5-4231-ac5e-767882e38ceb", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "ba8efa2f971d91bc6653c1258f739403579f86d87cc9b6063d7446084ab60f90", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "45852883-cdad-4b00-a25c-5444348663bd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/How_to_automate_S3_storage_with_functions.ipynb", "title": "How to automate tasks with functions (S3 bucket example)", "tokens": 2600, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "3298cbc60ac567678ff57c9d8182e02c7b68caa30ffb7eeb449bf0e257457fbb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10270, "end_char_idx": 12140, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
print(run_conversation('list my S3 buckets'))
```

You can ask the assistant to search for a specific file name either in all the buckets or in a specific one.


```python
search_file = '<file_name>'
print(run_conversation(f'search for a file {search_file} in all buckets'))
```


```python
search_word = '<file_name_part>'
bucket_name = '<bucket_name>'
print(run_conversation(f'search for a file contains {search_word} in {bucket_name}'))
```

The model is expected to clarify the ask from the user in case of ambiguity in the parameters values as described in the system message.


```python
print(run_conversation('search for a file'))
```

    Sure, to help me find what you're looking for, could you please provide the name of the file you want to search for and the name of the S3 bucket? Also, should the search match the file name exactly, or should it also consider partial matches?


#### Validate edge cases

We also instructed the model to reject irrelevant tasks. Let's test it out and see how it works in action.


```python
# the model should not answer details not related to the scope
print(run_conversation('what is the weather today'))
```

    Apologies for the misunderstanding, but I am only able to assist with S3 bucket functions. Can you please ask a question related to S3 bucket functions?


The provided functions are not limited to just retrieving information. They can also assist the user in uploading or downloading files.

#### Download a file


```python
search_file = '<file_name>'
bucket_name = '<bucket_name>'
local_directory = '<directory_path>'
print(run_conversation(f'download {search_file} from {bucket_name} bucket to {local_directory} directory'))
```

#### Upload a file


```python
local_file = '<file_name>'
bucket_name = '<bucket_name>'
print(run_conversation(f'upload {local_file} to {bucket_name} bucket'))
```


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb


key: _node_content
string_value:
{"id_": "3022f8d5-200f-4227-8b32-eaa355f99d66", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b2321cd7-c2a8-418f-b89d-8949e3474d3d", "node_type": "1", "metadata": {}, "hash": "99155a013f90dc7ac8d4103ce716eee1247240b046cb70f460ba84886c0e25ff", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2463, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Visualizing the embeddings in Kangas

In this Jupyter Notebook, we construct a Kangas DataGrid containing the data and projections of the embeddings into 2 dimensions.

## What is Kangas?

[Kangas](https://github.com/comet-ml/kangas/) as an open source, mixed-media, dataframe-like tool for data scientists. It was developed by [Comet](https://comet.com/), a company designed to help reduce the friction of moving models into production. 

### 1. Setup

To get started, we pip install kangas, and import it.


```python
%pip install kangas --quiet
```


```python
import kangas as kg
```

### 2. Constructing a Kangas DataGrid

We create a Kangas Datagrid with the original data and the embeddings. The data is composed of a rows of reviews, and the embeddings are composed of 1536 floating-point values. In this example, we get the data directly from github, in case you aren't running this notebook inside OpenAI's repo.

We use Kangas to read the CSV file into a DataGrid for further processing.


```python
data = kg.read_csv("https://raw.githubusercontent.com/openai/openai-cookbook/main/examples/data/fine_food_reviews_with_embeddings_1k.csv")
```

    Loading CSV file 'fine_food_reviews_with_embeddings_1k.csv'...


    1001it [00:00, 2412.90it/s]
    100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 2899.16it/s]


We can review the fields of the CSV file:


```python
data.info()
```

    DataGrid (in memory)
        Name   : fine_food_reviews_with_embeddings_1k
        Rows   : 1,000
        Columns: 9
    #   Column                Non-Null Count DataGrid Type       
    --- -------------------- --------------- --------------------
    1   Column 1                       1,000 INTEGER             
    2   ProductId                      1,000 TEXT                
    3   UserId                         1,000 TEXT                
    4   Score                          1,000 INTEGER             
    5   Summary                        1,000 TEXT                
    6   Text                           1,000 TEXT                
    7   combined                       1,000 TEXT                
    8   n_tokens                       1,000 INTEGER             
    9   embedding                      1,000 TEXT                


And get a glimpse of the first and last rows:


```python
data
```


key: _node_content
string_value:
{"id_": "b2321cd7-c2a8-418f-b89d-8949e3474d3d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3022f8d5-200f-4227-8b32-eaa355f99d66", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "eb01b02c99c76902db7d736d3b1f0a0d37b111a951907b0fb4411edab558466f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "759bf302-9680-464b-b63e-4bdf75863050", "node_type": "1", "metadata": {}, "hash": "6c56da3694b97000668891868af06ad4ef2cd5b6fc8871ebe728ccfe9bf05576", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1004, "end_char_idx": 3595, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
data = kg.read_csv("https://raw.githubusercontent.com/openai/openai-cookbook/main/examples/data/fine_food_reviews_with_embeddings_1k.csv")
```

    Loading CSV file 'fine_food_reviews_with_embeddings_1k.csv'...


    1001it [00:00, 2412.90it/s]
    100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:00<00:00, 2899.16it/s]


We can review the fields of the CSV file:


```python
data.info()
```

    DataGrid (in memory)
        Name   : fine_food_reviews_with_embeddings_1k
        Rows   : 1,000
        Columns: 9
    #   Column                Non-Null Count DataGrid Type       
    --- -------------------- --------------- --------------------
    1   Column 1                       1,000 INTEGER             
    2   ProductId                      1,000 TEXT                
    3   UserId                         1,000 TEXT                
    4   Score                          1,000 INTEGER             
    5   Summary                        1,000 TEXT                
    6   Text                           1,000 TEXT                
    7   combined                       1,000 TEXT                
    8   n_tokens                       1,000 INTEGER             
    9   embedding                      1,000 TEXT                


And get a glimpse of the first and last rows:


```python
data
```




<table><th colspan='1' >          row-id </th> <th colspan='1' >        Column 1 </th> <th colspan='1' >       ProductId </th> <th colspan='1' >          UserId </th> <th colspan='1' >           Score </th> <th colspan='1' >         Summary </th> <th colspan='1' >            Text </th> <th colspan='1' >        combined </th> <th colspan='1' >        n_tokens </th> <th colspan='1' >       embedding </th> <tr>
<td colspan='1' >               1 </td> <td colspan='1' >               0 </td> <td colspan='1' >      B003XPF9BO </td> <td colspan='1' >  A3R7JR3FMEBXQB </td> <td colspan='1' >               5 </td> <td colspan='1' > where does one  </td> <td colspan='1' > Wanted to save  </td> <td colspan='1' > Title: where do </td> <td colspan='1' >              52 </td> <td colspan='1' > [0.007018072064 </td> <tr>
<td colspan='1' >               2 </td> <td colspan='1' >             297 </td> <td colspan='1' >      B003VXHGPK </td> <td colspan='1' >  A21VWSCGW7UUAR </td> <td colspan='1' >               4 </td> <td colspan='1' > Good, but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good,


key: _node_content
string_value:
{"id_": "759bf302-9680-464b-b63e-4bdf75863050", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b2321cd7-c2a8-418f-b89d-8949e3474d3d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "97259f55fe468c8190a4131223366207f2bd4199bf592627cb7d897cfae0f239", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ac1659ae-9aa1-4dcd-abb9-e4f27032b0f7", "node_type": "1", "metadata": {}, "hash": "60bdff6a84fa0ce52bc6466d535d76a9881b8113a5d9f83f62075ba131967a28", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2468, "end_char_idx": 4477, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
<table><th colspan='1' >          row-id </th> <th colspan='1' >        Column 1 </th> <th colspan='1' >       ProductId </th> <th colspan='1' >          UserId </th> <th colspan='1' >           Score </th> <th colspan='1' >         Summary </th> <th colspan='1' >            Text </th> <th colspan='1' >        combined </th> <th colspan='1' >        n_tokens </th> <th colspan='1' >       embedding </th> <tr>
<td colspan='1' >               1 </td> <td colspan='1' >               0 </td> <td colspan='1' >      B003XPF9BO </td> <td colspan='1' >  A3R7JR3FMEBXQB </td> <td colspan='1' >               5 </td> <td colspan='1' > where does one  </td> <td colspan='1' > Wanted to save  </td> <td colspan='1' > Title: where do </td> <td colspan='1' >              52 </td> <td colspan='1' > [0.007018072064 </td> <tr>
<td colspan='1' >               2 </td> <td colspan='1' >             297 </td> <td colspan='1' >      B003VXHGPK </td> <td colspan='1' >  A21VWSCGW7UUAR </td> <td colspan='1' >               4 </td> <td colspan='1' > Good, but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good, bu </td> <td colspan='1' >             178 </td> <td colspan='1' > [-0.00314055196 </td> <tr>
<td colspan='1' >               3 </td> <td colspan='1' >             296 </td> <td colspan='1' >      B008JKTTUA </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>
<td colspan='1' >               4 </td> <td colspan='1' >             295 </td> <td colspan='1' >      B000LKTTTW </td> <td colspan='1' >  A14MQ40CCU8B13 </td> <td colspan='1' >               5 </td> <td colspan='1' > Best tomato sou </td> <td colspan='1' > I have a hard t </td> <td colspan='1' > Title: Best tom </td> <td colspan='1' >             111 </td> <td colspan='1' > [-0.


key: _node_content
string_value:
{"id_": "ac1659ae-9aa1-4dcd-abb9-e4f27032b0f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "759bf302-9680-464b-b63e-4bdf75863050", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "29b10349a3fcb01e5cf774d02bb2604090808094eae77661698caffdd504cd15", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ec4be264-77c0-43c2-91de-230bc5c976ae", "node_type": "1", "metadata": {}, "hash": "969c92fd9b6fae0ac217144b2ec99033930de793a03be0b417be601d85f7c386", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
but not W </td> <td colspan='1' > Honestly, I hav </td> <td colspan='1' > Title: Good, bu </td> <td colspan='1' >             178 </td> <td colspan='1' > [-0.00314055196 </td> <tr>
<td colspan='1' >               3 </td> <td colspan='1' >             296 </td> <td colspan='1' >      B008JKTTUA </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>
<td colspan='1' >               4 </td> <td colspan='1' >             295 </td> <td colspan='1' >      B000LKTTTW </td> <td colspan='1' >  A14MQ40CCU8B13 </td> <td colspan='1' >               5 </td> <td colspan='1' > Best tomato sou </td> <td colspan='1' > I have a hard t </td> <td colspan='1' > Title: Best tom </td> <td colspan='1' >             111 </td> <td colspan='1' > [-0.00139322795 </td> <tr>
<td colspan='1' >               5 </td> <td colspan='1' >             294 </td> <td colspan='1' >      B001D09KAM </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>
<tr><td colspan='10' style='text-align: left;'>.</td></tr><td colspan='1' >             996 </td> <td colspan='1' >             623 </td> <td colspan='1' >      B0000CFXYA </td> <td colspan='1' >  A3GS4GWPIBV0NT </td> <td colspan='1' >               1 </td> <td colspan='1' > Strange inflamm </td> <td colspan='1' > Truthfully wasn </td> <td colspan='1' > Title: Strange  </td> <td colspan='1' >             110 </td> <td colspan='1' > [0.


key: _node_content
string_value:
{"id_": "ec4be264-77c0-43c2-91de-230bc5c976ae", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ac1659ae-9aa1-4dcd-abb9-e4f27032b0f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "04e066931487bf070cf4257b20244c18df149aae2f1a640bb71f893ca54fd28a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "300eb047-8096-4a13-a92c-654c098b7328", "node_type": "1", "metadata": {}, "hash": "2c3908d5c2f433ac619ad62af16886513f591776e8aa55e2df8f69a05ee059ba", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": null, "end_char_idx": null, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
00139322795 </td> <tr>
<td colspan='1' >               5 </td> <td colspan='1' >             294 </td> <td colspan='1' >      B001D09KAM </td> <td colspan='1' >  A34XBAIFT02B60 </td> <td colspan='1' >               1 </td> <td colspan='1' > Should advertis </td> <td colspan='1' > First, these sh </td> <td colspan='1' > Title: Should a </td> <td colspan='1' >              78 </td> <td colspan='1' > [-0.01757248118 </td> <tr>
<tr><td colspan='10' style='text-align: left;'>.</td></tr><td colspan='1' >             996 </td> <td colspan='1' >             623 </td> <td colspan='1' >      B0000CFXYA </td> <td colspan='1' >  A3GS4GWPIBV0NT </td> <td colspan='1' >               1 </td> <td colspan='1' > Strange inflamm </td> <td colspan='1' > Truthfully wasn </td> <td colspan='1' > Title: Strange  </td> <td colspan='1' >             110 </td> <td colspan='1' > [0.000110913533 </td> <tr>
<td colspan='1' >             997 </td> <td colspan='1' >             624 </td> <td colspan='1' >      B0001BH5YM </td> <td colspan='1' >   A1BZ3HMAKK0NC </td> <td colspan='1' >               5 </td> <td colspan='1' > My favorite and </td> <td colspan='1' > You've just got </td> <td colspan='1' > Title: My favor </td> <td colspan='1' >              80 </td> <td colspan='1' > [-0.02086931467 </td> <tr>
<td colspan='1' >             998 </td> <td colspan='1' >             625 </td> <td colspan='1' >      B0009ET7TC </td> <td colspan='1' >  A2FSDQY5AI6TNX </td> <td colspan='1' >               5 </td> <td colspan='1' > My furbabies LO </td> <td colspan='1' > Shake the conta </td> <td colspan='1' > Title: My furba </td> <td colspan='1' >              47 </td> <td colspan='1' > [-0.00974910240 </td> <tr>
<td colspan='1' >             999 </td> <td colspan='1' >             619 </td> <td colspan='1' >      B007PA32L2 </td> <td colspan='1' >  A15FF2P7RPKH6G </td> <td colspan='1' >               5 </td> <td colspan='1' > got this for th </td> <td colspan='1' > all i have hear </td> <td colspan='1' > Title: got this </td> <td colspan='1' >              50 </td> <td colspan='1' > [-0.


key: _node_content
string_value:
{"id_": "300eb047-8096-4a13-a92c-654c098b7328", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ec4be264-77c0-43c2-91de-230bc5c976ae", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "e58befb2223c57c1766a4623659097b15b738722d517d8ff7979910ed7b6c37e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "712fe3a9-6b26-44e6-b7a2-ce6d2ae77c22", "node_type": "1", "metadata": {}, "hash": "b4315d16f37dcefbee10397aa1bfc1df2ed127ec7827285bbfc01c6dbcb42153", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5752, "end_char_idx": 7916, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
02086931467 </td> <tr>
<td colspan='1' >             998 </td> <td colspan='1' >             625 </td> <td colspan='1' >      B0009ET7TC </td> <td colspan='1' >  A2FSDQY5AI6TNX </td> <td colspan='1' >               5 </td> <td colspan='1' > My furbabies LO </td> <td colspan='1' > Shake the conta </td> <td colspan='1' > Title: My furba </td> <td colspan='1' >              47 </td> <td colspan='1' > [-0.00974910240 </td> <tr>
<td colspan='1' >             999 </td> <td colspan='1' >             619 </td> <td colspan='1' >      B007PA32L2 </td> <td colspan='1' >  A15FF2P7RPKH6G </td> <td colspan='1' >               5 </td> <td colspan='1' > got this for th </td> <td colspan='1' > all i have hear </td> <td colspan='1' > Title: got this </td> <td colspan='1' >              50 </td> <td colspan='1' > [-0.00521062919 </td> <tr>
<td colspan='1' >            1000 </td> <td colspan='1' >             999 </td> <td colspan='1' >      B001EQ5GEO </td> <td colspan='1' >  A3VYU0VO6DYV6I </td> <td colspan='1' >               5 </td> <td colspan='1' > I love Maui Cof </td> <td colspan='1' > My first experi </td> <td colspan='1' > Title: I love M </td> <td colspan='1' >             118 </td> <td colspan='1' > [-0.00605782261 </td> <tr>
<tr>
<td colspan='10' style="text-align: left;"> [1000 rows x 9 columns] </td> <tr>
<tr><td colspan='10' style='text-align: left;'></td></tr><tr><td colspan='10' style='text-align: left;'>*  Use DataGrid.save() to save to disk</td></tr><tr><td colspan='10' style='text-align: left;'>** Use DataGrid.show() to start user interface</td></tr></table>



Now, we create a new DataGrid, converting the numbers into an Embedding:


```python
import ast # to convert string of a list of numbers into a list of numbers

dg = kg.DataGrid(
    name="openai_embeddings",
    columns=data.get_columns(),
    converters={"Score": str},
)
for row in data:
    embedding = ast.literal_eval(row[8])
    row[8] = kg.Embedding(
        embedding, 
        name=str(row[3]), 
        text="%s - %.10s" % (row[3], row[4]),
        projection="umap",
    )
    dg.append(row)
```

The new DataGrid now has an Embedding column with proper datatype.


key: _node_content
string_value:
{"id_": "712fe3a9-6b26-44e6-b7a2-ce6d2ae77c22", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b465e9de-7e09-4654-bbe8-5b49f0846fb0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "75a89f006f1de311679cb3df02eccc11236f8ebf85f52e9cfcb503a35e467985", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "300eb047-8096-4a13-a92c-654c098b7328", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_in_Kangas.ipynb", "title": "## Visualizing the embeddings in Kangas", "tokens": 2958, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "eb963b02af751b225cedd8b827386685a73933f34fcafcebec17432d9f5acd3a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6967, "end_char_idx": 9839, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
00605782261 </td> <tr>
<tr>
<td colspan='10' style="text-align: left;"> [1000 rows x 9 columns] </td> <tr>
<tr><td colspan='10' style='text-align: left;'></td></tr><tr><td colspan='10' style='text-align: left;'>*  Use DataGrid.save() to save to disk</td></tr><tr><td colspan='10' style='text-align: left;'>** Use DataGrid.show() to start user interface</td></tr></table>



Now, we create a new DataGrid, converting the numbers into an Embedding:


```python
import ast # to convert string of a list of numbers into a list of numbers

dg = kg.DataGrid(
    name="openai_embeddings",
    columns=data.get_columns(),
    converters={"Score": str},
)
for row in data:
    embedding = ast.literal_eval(row[8])
    row[8] = kg.Embedding(
        embedding, 
        name=str(row[3]), 
        text="%s - %.10s" % (row[3], row[4]),
        projection="umap",
    )
    dg.append(row)
```

The new DataGrid now has an Embedding column with proper datatype.


```python
dg.info()
```

    DataGrid (in memory)
        Name   : openai_embeddings
        Rows   : 1,000
        Columns: 9
    #   Column                Non-Null Count DataGrid Type       
    --- -------------------- --------------- --------------------
    1   Column 1                       1,000 INTEGER             
    2   ProductId                      1,000 TEXT                
    3   UserId                         1,000 TEXT                
    4   Score                          1,000 TEXT                
    5   Summary                        1,000 TEXT                
    6   Text                           1,000 TEXT                
    7   combined                       1,000 TEXT                
    8   n_tokens                       1,000 INTEGER             
    9   embedding                      1,000 EMBEDDING-ASSET     


We simply save the datagrid, and we're done.


```python
dg.save()
```

### 3. Render 2D Projections

To render the data directly in the notebook, simply show it. Note that each row contains an embedding projection. 

Scroll to far right to see embeddings projection per row.

The color of the point in projection space represents the Score.


```python
dg.show()
```



<iframe
    width="100%"
    height="750px"
    src="http://127.0.1.1:4000/?datagrid=openai_embeddings.datagrid&timestamp=1685559502.7515423"
    frameborder="0"
    allowfullscreen

></iframe>



Group by "Score" to see rows of each group.


```python
dg.show(group="Score", sort="Score", rows=5, select="Score,embedding")
```



<iframe
    width="100%"
    height="750px"
    src="http://127.0.1.1:4000/?datagrid=openai_embeddings.datagrid&timestamp=1685559502.7515423&group=Score&sort=Score&rows=5&select=Score%2Cembedding"
    frameborder="0"
    allowfullscreen

></iframe>



An example of this datagrid is hosted here: https://kangas.comet.com/?datagrid=/data/openai_embeddings.datagrid


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb


key: _node_content
string_value:
{"id_": "9c48f6e2-4d2c-4be7-9fdf-56a4fc6d1e86", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "685f49fa-bbe1-40e0-b498-8bc00d781e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "26a4c0e3-60e0-4424-9600-3ea84b35dda8", "node_type": "1", "metadata": {}, "hash": "58041455514a08016ed1c06ea7a8cc66a5f924abf18ed23fec08050c5b4f5ae4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 1294, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Visualizing Open AI Embeddings in Atlas

In this example, we will upload food review embeddings to [Atlas](https://atlas.nomic.ai) to visualize the embeddings.

## What is Atlas?

[Atlas](https://atlas.nomic.ai) is a machine learning tool used to visualize massive datasets of embeddings in your web browser. Upload millions of embeddings to Atlas and interact with them in your web browser or jupyter notebook.

### 1. Login to Atlas.



```python
!pip install nomic
```

    


```python
import pandas as pd
import numpy as np
from ast import literal_eval

# Load the embeddings
datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"
df = pd.read_csv(datafile_path)

# Convert to a list of lists of floats
embeddings = np.array(df.embedding.apply(literal_eval).to_list())
df = df.drop('embedding', axis=1)
df = df.rename(columns={'Unnamed: 0': 'id'})

```


```python
import nomic
from nomic import atlas
nomic.login('7xDPkYXSYDc1_ErdTPIcoAR9RNd8YDlkS3nVNXcVoIMZ6') #demo account

data = df.to_dict('records')
project = atlas.map_embeddings(embeddings=embeddings, data=data,
                               id_field='id',
                               colorable_fields=['Score'])
map = project.maps[0]
```

    

### 2. Interact with your embeddings in Jupyter


```python
map
```


key: _node_content
string_value:
{"id_": "26a4c0e3-60e0-4424-9600-3ea84b35dda8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "685f49fa-bbe1-40e0-b498-8bc00d781e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9c48f6e2-4d2c-4be7-9fdf-56a4fc6d1e86", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "b58f9e9568ae05ced34c2ee368834e0b53b031b922c4c41dcdb7caa75696a214", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "37d783ca-f359-4866-b383-3681b8f3a5d2", "node_type": "1", "metadata": {}, "hash": "5b0d7e8666a5b807e3bbea3400b6775cad72d42af45bacc0f92b52f9d959d8db", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2557, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Visualizing Open AI Embeddings in Atlas

In this example, we will upload food review embeddings to [Atlas](https://atlas.nomic.ai) to visualize the embeddings.

## What is Atlas?

[Atlas](https://atlas.nomic.ai) is a machine learning tool used to visualize massive datasets of embeddings in your web browser. Upload millions of embeddings to Atlas and interact with them in your web browser or jupyter notebook.

### 1. Login to Atlas.



```python
!pip install nomic
```

    


```python
import pandas as pd
import numpy as np
from ast import literal_eval

# Load the embeddings
datafile_path = "data/fine_food_reviews_with_embeddings_1k.csv"
df = pd.read_csv(datafile_path)

# Convert to a list of lists of floats
embeddings = np.array(df.embedding.apply(literal_eval).to_list())
df = df.drop('embedding', axis=1)
df = df.rename(columns={'Unnamed: 0': 'id'})

```


```python
import nomic
from nomic import atlas
nomic.login('7xDPkYXSYDc1_ErdTPIcoAR9RNd8YDlkS3nVNXcVoIMZ6') #demo account

data = df.to_dict('records')
project = atlas.map_embeddings(embeddings=embeddings, data=data,
                               id_field='id',
                               colorable_fields=['Score'])
map = project.maps[0]
```

    

### 2. Interact with your embeddings in Jupyter


```python
map
```





    <h3>Project: meek-laborer</h3>
    <script>
    destroy = function() {
        document.getElementById("iframe463f4614-7689-47e4-b55b-1da0cc679559").remove()
    }
</script>

<h4>Projection ID: 463f4614-7689-47e4-b55b-1da0cc679559</h4>
<div class="actions">
    <div id="hide" class="action" onclick="destroy()">Hide embedded project</div>
    <div class="action" id="out">
        <a href="https://atlas.nomic.ai/map/fddc0e07-97c5-477c-827c-96bca44519aa/463f4614-7689-47e4-b55b-1da0cc679559" target="_blank">Explore on atlas.nomic.ai</a>
    </div>
</div>

<iframe class="iframe" id="iframe463f4614-7689-47e4-b55b-1da0cc679559" allow="clipboard-read; clipboard-write" src="https://atlas.nomic.ai/map/fddc0e07-97c5-477c-827c-96bca44519aa/463f4614-7689-47e4-b55b-1da0cc679559">
</iframe>

<style>
    .iframe {
        /* vh can be **very** large in vscode ipynb. */
        height: min(75vh, 66vw);
        width: 100%;
    }
</style>

<style>
    .actions {
      display: block;
    }
    .action {
      min-height: 18px;
      margin: 5px;
      transition: all 500ms ease-in-out;
    }
    .action:hover {
      cursor: pointer;
    }
    #hide:hover::after {
        content: " X";
    }
    #out:hover::after {
        content: "";
    }
</style>


key: _node_content
string_value:
{"id_": "37d783ca-f359-4866-b383-3681b8f3a5d2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "685f49fa-bbe1-40e0-b498-8bc00d781e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "995be937ab9d1625166e0acc01b9da3da726385e19c199303d65a3ab24962083", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "26a4c0e3-60e0-4424-9600-3ea84b35dda8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/Visualizing_embeddings_with_Atlas.ipynb", "title": "Load the embeddings", "tokens": 776, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "729cb66d6a90668ef647874d4b2d45bedb78243aae102e75538732b69c6eaa99", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2564, "end_char_idx": 2578, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb


key: _node_content
string_value:
{"id_": "0b26cc28-595c-4851-9d16-845e8e988f8a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "40549817-9a21-4375-8e72-788f72ed5392", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c6c45efa9eb125046624692b393e51c3ff26792f1c25ae7dc0550d744537b171", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b725042a-f1f1-4680-9b71-8ea52162fabf", "node_type": "1", "metadata": {}, "hash": "fb1e8c8c2c5bec55ae22dc6ecb3ccb784c51d2c593b1464d455ae73ef3fff809", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3123, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Financial Document Analysis with LlamaIndex

In this example notebook, we showcase how to perform financial analysis over [**10-K**](https://en.wikipedia.org/wiki/Form_10-K) documents with the [**LlamaIndex**](https://gpt-index.readthedocs.io/en/latest/) framework with just a few lines of code.

## Notebook Outline
* [Introduction](#Introduction)
* [Setup](#Setup)
* [Data Loading & Indexing](#Data-Loading-and-Indexing)
* [Simple QA](#Simple-QA)
* [Advanced QA - Compare and Contrast](#Advanced-QA---Compare-and-Contrast)


## Introduction

### LLamaIndex
[LlamaIndex](https://gpt-index.readthedocs.io/en/latest/) is a data framework for LLM applications.
You can get started with just a few lines of code and build a retrieval-augmented generation (RAG) system in minutes.
For more advanced users, LlamaIndex offers a rich toolkit for ingesting and indexing your data, modules for retrieval and re-ranking, and composable components for building custom query engines.

See [full documentation](https://gpt-index.readthedocs.io/en/latest/) for more details.

### Financial Analysis over 10-K documents
A key part of a financial analyst's job is to extract information and synthesize insight from long financial documents.
A great example is the 10-K form - an annual report required by the U.S. Securities and Exchange Commission (SEC), that gives a comprehensive summary of a company's financial performance.
These documents typically run hundred of pages in length, and contain domain-specific terminology that makes it challenging for a layperson to digest quickly.


We showcase how LlamaIndex can support a financial analyst in quickly extracting information and synthesize insights **across multiple documents** with very little coding. 

## Setup

To begin, we need to install the llama-index library


```python
!pip install llama-index pypdf
```

Now, we import all modules used in this tutorial


```python
from langchain import OpenAI

from llama_index import SimpleDirectoryReader, ServiceContext, VectorStoreIndex
from llama_index import set_global_service_context
from llama_index.response.pprint_utils import pprint_response
from llama_index.tools import QueryEngineTool, ToolMetadata
from llama_index.query_engine import SubQuestionQueryEngine
```

Before we start, we can configure the LLM provider and model that will power our RAG system.  
Here, we pick `gpt-3.5-turbo-instruct` from OpenAI.  


```python
llm = OpenAI(temperature=0, model_name="gpt-3.5-turbo-instruct", max_tokens=-1)
```

We construct a `ServiceContext` and set it as the global default, so all subsequent operations that depends on LLM calls will use the model we configured here.


```python
service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context=service_context)
```

## Data Loading and Indexing

Now, we load and parse 2 PDFs (one for Uber 10-K in 2021 and another for Lyft 10-k in 2021).    
Under the hood, the PDFs are converted to plain text `Document` objects, separate by page.  

> Note: this operation might take a while to run, since each document is more than 100 pages.


key: _node_content
string_value:
{"id_": "b725042a-f1f1-4680-9b71-8ea52162fabf", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "40549817-9a21-4375-8e72-788f72ed5392", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c6c45efa9eb125046624692b393e51c3ff26792f1c25ae7dc0550d744537b171", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "0b26cc28-595c-4851-9d16-845e8e988f8a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "2250595aadff75d1139313acaa93d9c62def9b82f2d44fcc485e3f1c3b81773a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "03ad2555-8db0-42a9-9b71-2f479e635244", "node_type": "1", "metadata": {}, "hash": "32f63c194005533c3a439a19e2ed545c88309c0df45aa52f20d5c8f903591004", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1576, "end_char_idx": 4689, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
We showcase how LlamaIndex can support a financial analyst in quickly extracting information and synthesize insights **across multiple documents** with very little coding. 

## Setup

To begin, we need to install the llama-index library


```python
!pip install llama-index pypdf
```

Now, we import all modules used in this tutorial


```python
from langchain import OpenAI

from llama_index import SimpleDirectoryReader, ServiceContext, VectorStoreIndex
from llama_index import set_global_service_context
from llama_index.response.pprint_utils import pprint_response
from llama_index.tools import QueryEngineTool, ToolMetadata
from llama_index.query_engine import SubQuestionQueryEngine
```

Before we start, we can configure the LLM provider and model that will power our RAG system.  
Here, we pick `gpt-3.5-turbo-instruct` from OpenAI.  


```python
llm = OpenAI(temperature=0, model_name="gpt-3.5-turbo-instruct", max_tokens=-1)
```

We construct a `ServiceContext` and set it as the global default, so all subsequent operations that depends on LLM calls will use the model we configured here.


```python
service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context=service_context)
```

## Data Loading and Indexing

Now, we load and parse 2 PDFs (one for Uber 10-K in 2021 and another for Lyft 10-k in 2021).    
Under the hood, the PDFs are converted to plain text `Document` objects, separate by page.  

> Note: this operation might take a while to run, since each document is more than 100 pages.


```python
lyft_docs = SimpleDirectoryReader(input_files=["../data/10k/lyft_2021.pdf"]).load_data()
uber_docs = SimpleDirectoryReader(input_files=["../data/10k/uber_2021.pdf"]).load_data()
```


```python
print(f'Loaded lyft 10-K with {len(lyft_docs)} pages')
print(f'Loaded Uber 10-K with {len(uber_docs)} pages')
```

    Loaded lyft 10-K with 238 pages
    Loaded Uber 10-K with 307 pages


Now, we can build an (in-memory) `VectorStoreIndex` over the documents that we've loaded.  

> Note: this operation might take a while to run, since it calls OpenAI API for computing vector embedding over document chunks.


```python
lyft_index = VectorStoreIndex.from_documents(lyft_docs)
uber_index = VectorStoreIndex.from_documents(uber_docs)
```

## Simple QA

Now we are ready to run some queries against our indices!  
To do so, we first configure a `QueryEngine`, which just captures a set of configurations for how we want to query the underlying index.

For a `VectorStoreIndex`, the most common configuration to adjust is `similarity_top_k` which controls how many document chunks (which we call `Node` objects) are retrieved to use as context for answering our question.


```python
lyft_engine = lyft_index.as_query_engine(similarity_top_k=3)
```


```python
uber_engine = uber_index.as_query_engine(similarity_top_k=3)
```

Let's see some queries in action!


```python
response = await lyft_engine.aquery('What is the revenue of Lyft in 2021? Answer in millions with page reference')
```


```python
print(response)
```

    
    $3,208.3 million (page 63)


key: _node_content
string_value:
{"id_": "03ad2555-8db0-42a9-9b71-2f479e635244", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "40549817-9a21-4375-8e72-788f72ed5392", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c6c45efa9eb125046624692b393e51c3ff26792f1c25ae7dc0550d744537b171", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b725042a-f1f1-4680-9b71-8ea52162fabf", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "d8ae4af8a52ad9359c05738dcf405e241b747d79b5675f755deb5172f8330a94", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "37254ed0-8592-4923-91b5-441606eceee5", "node_type": "1", "metadata": {}, "hash": "d5d2afdb820fa2a38202a5e850a5e4ba352e37c2ec00f12ece24f47bf845decb", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3126, "end_char_idx": 5855, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
lyft_docs = SimpleDirectoryReader(input_files=["../data/10k/lyft_2021.pdf"]).load_data()
uber_docs = SimpleDirectoryReader(input_files=["../data/10k/uber_2021.pdf"]).load_data()
```


```python
print(f'Loaded lyft 10-K with {len(lyft_docs)} pages')
print(f'Loaded Uber 10-K with {len(uber_docs)} pages')
```

    Loaded lyft 10-K with 238 pages
    Loaded Uber 10-K with 307 pages


Now, we can build an (in-memory) `VectorStoreIndex` over the documents that we've loaded.  

> Note: this operation might take a while to run, since it calls OpenAI API for computing vector embedding over document chunks.


```python
lyft_index = VectorStoreIndex.from_documents(lyft_docs)
uber_index = VectorStoreIndex.from_documents(uber_docs)
```

## Simple QA

Now we are ready to run some queries against our indices!  
To do so, we first configure a `QueryEngine`, which just captures a set of configurations for how we want to query the underlying index.

For a `VectorStoreIndex`, the most common configuration to adjust is `similarity_top_k` which controls how many document chunks (which we call `Node` objects) are retrieved to use as context for answering our question.


```python
lyft_engine = lyft_index.as_query_engine(similarity_top_k=3)
```


```python
uber_engine = uber_index.as_query_engine(similarity_top_k=3)
```

Let's see some queries in action!


```python
response = await lyft_engine.aquery('What is the revenue of Lyft in 2021? Answer in millions with page reference')
```


```python
print(response)
```

    
    $3,208.3 million (page 63)



```python
response = await uber_engine.aquery('What is the revenue of Uber in 2021? Answer in millions, with page reference')
```


```python
print(response)
```

    
    $17,455 (page 53)


## Advanced QA - Compare and Contrast

For more complex financial analysis, one often needs to reference multiple documents.  

As a example, let's take a look at how to do compare-and-contrast queries over both Lyft and Uber financials.  
For this, we build a `SubQuestionQueryEngine`, which breaks down a complex compare-and-contrast query, into simpler sub-questions to execute on respective sub query engine backed by individual indices.


```python
query_engine_tools = [
    QueryEngineTool(
        query_engine=lyft_engine, 
        metadata=ToolMetadata(name='lyft_10k', description='Provides information about Lyft financials for year 2021')
    ),
    QueryEngineTool(
        query_engine=uber_engine, 
        metadata=ToolMetadata(name='uber_10k', description='Provides information about Uber financials for year 2021')
    ),
]

s_engine = SubQuestionQueryEngine.from_defaults(query_engine_tools=query_engine_tools)
```

Let's see these queries in action!


key: _node_content
string_value:
{"id_": "37254ed0-8592-4923-91b5-441606eceee5", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "40549817-9a21-4375-8e72-788f72ed5392", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c6c45efa9eb125046624692b393e51c3ff26792f1c25ae7dc0550d744537b171", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "03ad2555-8db0-42a9-9b71-2f479e635244", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "7054c28ed1def82986d85e6f63a25ef6ca56f8c21d9496e3113491a06f663852", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e88d1194-09a5-4acb-ab57-864b9625d515", "node_type": "1", "metadata": {}, "hash": "51a1b374e2f803bfacc0f73c739c33b50edadbd7cf22bfea926744b34de7080f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4303, "end_char_idx": 7987, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
lyft_engine = lyft_index.as_query_engine(similarity_top_k=3)
```


```python
uber_engine = uber_index.as_query_engine(similarity_top_k=3)
```

Let's see some queries in action!


```python
response = await lyft_engine.aquery('What is the revenue of Lyft in 2021? Answer in millions with page reference')
```


```python
print(response)
```

    
    $3,208.3 million (page 63)



```python
response = await uber_engine.aquery('What is the revenue of Uber in 2021? Answer in millions, with page reference')
```


```python
print(response)
```

    
    $17,455 (page 53)


## Advanced QA - Compare and Contrast

For more complex financial analysis, one often needs to reference multiple documents.  

As a example, let's take a look at how to do compare-and-contrast queries over both Lyft and Uber financials.  
For this, we build a `SubQuestionQueryEngine`, which breaks down a complex compare-and-contrast query, into simpler sub-questions to execute on respective sub query engine backed by individual indices.


```python
query_engine_tools = [
    QueryEngineTool(
        query_engine=lyft_engine, 
        metadata=ToolMetadata(name='lyft_10k', description='Provides information about Lyft financials for year 2021')
    ),
    QueryEngineTool(
        query_engine=uber_engine, 
        metadata=ToolMetadata(name='uber_10k', description='Provides information about Uber financials for year 2021')
    ),
]

s_engine = SubQuestionQueryEngine.from_defaults(query_engine_tools=query_engine_tools)
```

Let's see these queries in action!


```python
response = await s_engine.aquery('Compare and contrast the customer segments and geographies that grew the fastest')
```

    Generated 4 sub questions.
    [36;1m[1;3m[uber_10k] Q: What customer segments grew the fastest for Uber
    [0m[36;1m[1;3m[uber_10k] A: in 2021?
    
    The customer segments that grew the fastest for Uber in 2021 were its Mobility Drivers, Couriers, Riders, and Eaters. These segments experienced growth due to the continued stay-at-home order demand related to COVID-19, as well as Uber's introduction of its Uber One, Uber Pass, Eats Pass, and Rides Pass membership programs. Additionally, Uber's marketplace-centric advertising helped to connect merchants and brands with its platform network, further driving growth.
    [0m[33;1m[1;3m[uber_10k] Q: What geographies grew the fastest for Uber
    [0m[33;1m[1;3m[uber_10k] A: 
    Based on the context information, it appears that Uber experienced the most growth in large metropolitan areas, such as Chicago, Miami, New York City, Sao Paulo, and London. Additionally, Uber experienced growth in suburban and rural areas, as well as in countries such as Argentina, Germany, Italy, Japan, South Korea, and Spain.
    [0m[38;5;200m[1;3m[lyft_10k] Q: What customer segments grew the fastest for Lyft
    [0m[38;5;200m[1;3m[lyft_10k] A: 
    The customer segments that grew the fastest for Lyft were ridesharing, light vehicles, and public transit. Ridesharing grew as Lyft was able to predict demand and proactively incentivize drivers to be available for rides in the right place at the right time. Light vehicles grew as users were looking for options that were more active, usually lower-priced, and often more efficient for short trips during heavy traffic. Public transit grew as Lyft integrated third-party public transit data into the Lyft App to offer users a robust view of transportation options around them.
    [0m[32;1m[1;3m[lyft_10k] Q: What geographies grew the fastest for Lyft
    [0m[32;1m[1;3m[lyft_10k] A: 
    It is not possible to answer this question with the given context information.
    [0m


key: _node_content
string_value:
{"id_": "e88d1194-09a5-4acb-ab57-864b9625d515", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "40549817-9a21-4375-8e72-788f72ed5392", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "c6c45efa9eb125046624692b393e51c3ff26792f1c25ae7dc0550d744537b171", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "37254ed0-8592-4923-91b5-441606eceee5", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/third_party/financial_document_analysis_with_llamaindex.ipynb", "title": "Financial Document Analysis with LlamaIndex", "tokens": 2638, "retrieve_doc": true, "source": "openai_cookbooks"}, "hash": "a71d8cfafbb8a73e35af5b561ae16135a807711966e3e1f2d3170bdca7b40ad3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7990, "end_char_idx": 10896, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


